Prototype - Python frames

CMakeLists.txt

@@ -264,6 +264,14 @@ target_link_libraries(
 )
 target_link_libraries(dd_profiling-embedded PUBLIC dl pthread rt)
 
+# add libaustin
+find_library(austin_location NAMES libaustin.a)
+message(STATUS "libaustin location at:" ${austin_location})
+add_library(austin STATIC IMPORTED)
+set_target_properties(
+  austin PROPERTIES IMPORTED_LOCATION ${austin_location}
+                    INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libaustin")
+
 set(LIBDD_PROFILING_EMBEDDED_OBJECT "${CMAKE_BINARY_DIR}/libdd_profiling-embedded.o")
 add_custom_command(
   OUTPUT ${LIBDD_PROFILING_EMBEDDED_OBJECT}
@@ -283,6 +291,10 @@ add_exe(
   LIBRARIES ${DDPROF_LIBRARY_LIST}
   DEFINITIONS ${DDPROF_DEFINITION_LIST})
 target_link_libraries(ddprof PRIVATE libddprofiling_embedded_object)
+
+# link libaustin to ddprof
+target_link_libraries(ddprof PRIVATE austin)
+
 if(USE_LOADER)
   target_compile_definitions(ddprof PRIVATE "DDPROF_USE_LOADER")
   target_link_libraries(ddprof PRIVATE libdd_loader_object)

app/base-env/Dockerfile

@@ -177,6 +177,22 @@ RUN VERSION="3.21.0" \
 # Install cmake_format
 RUN pip3 install cmake_format
 
+RUN apt-get update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    libtool \
+    libiberty-dev
+
+
+# possibly useful command
+# CFLAGS="-g -O0 -DTRACE" ./configure
+# Valid commit : git checkout a7a292b3f5a1058051b017c0d339a678efdec704
+RUN git clone --branch libaustin https://github.com/P403n1x87/austin.git && \
+    cd austin && \
+    autoreconf --install && \
+    CFLAGS="-O2" ./configure && \
+    make && \
+    make install
+
 # A specific user is required to get access to perf event ressources.
 # This enables unit testing using perf-event ressources
 RUN useradd -ms /bin/bash ddbuild

app/python-tests/Dockerfile

@@ -0,0 +1,2 @@
+FROM python:slim
+

cmake/Findelfutils.cmake

@@ -28,6 +28,9 @@ set(LIBELF_PATH ${ELFUTILS_PATH}/lib/libelf.a)
 
 set(ELFUTILS_INCLUDE_DIRS ${ELFUTILS_PATH}/include)
 
+set(PRIVATE_EBL_DIRS ${ELFUTILS_PATH}/src/libebl)
+set(LIBDW_INCLUDE_DIRS ${ELFUTILS_PATH}/include/elfutils)
+
 if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Release" AND NOT "${CMAKE_BUILD_TYPE}" STREQUAL
                                                     "RelWithDebInfo")
   # Variable can contain several args as it is quoted
@@ -59,5 +62,8 @@ set_target_properties(
              INTERFACE_INCLUDE_DIRECTORIES ${ELFUTILS_INCLUDE_DIRS}
              INTERFACE_LINK_LIBRARIES "${LIBLZMA_LIBRARIES};${ZLIB_LIBRARIES}")
 
+add_library(private_ebl INTERFACE)
+target_include_directories(private_ebl INTERFACE ${PRIVATE_EBL_DIRS} ${LIBDW_INCLUDE_DIRS})
+
 # Elf libraries
-set(ELFUTILS_LIBRARIES dw elf)
+set(ELFUTILS_LIBRARIES dw elf private_ebl)

include/austin_symbol_lookup.hpp

@@ -0,0 +1,31 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#pragma once
+
+#include "symbol_table.hpp"
+
+#include "libaustin.h"
+
+#include <unordered_map>
+
+namespace ddprof {
+
+class AustinSymbolLookup {
+public:
+  SymbolIdx_t get_or_insert(austin_frame_t *frame, SymbolTable &symbol_table);
+  void clear() {
+    for (auto el : _austin_symbols) {
+      _free_list.push_back(el);
+    }
+    _austin_symbols.clear();
+  }
+
+private:
+  std::vector<SymbolIdx_t> _austin_symbols;
+  std::vector<SymbolIdx_t> _free_list;
+};
+
+} // namespace ddprof

include/ddprof_process.hpp

@@ -13,13 +13,16 @@
 #include <limits>
 #include <sys/types.h>
 #include <unordered_map>
+#include <vector>
+
+typedef void *austin_handle_t;
 
 namespace ddprof {
 
 class Process {
 public:
   explicit Process(pid_t pid) : _pid(pid), _cgroup_ns(kCGroupNsNull) {}
-
+  ~Process();
   using CGroupId_t = uint64_t;
   static constexpr CGroupId_t kCGroupNsNull =
       std::numeric_limits<CGroupId_t>::max();
@@ -33,8 +36,12 @@ class Process {
   // lazy read of container id
   const ContainerId &get_container_id(std::string_view path_to_proc = "");
 
+  austin_handle_t get_austin_handle(pid_t tid);
+
   uint64_t _sample_counter = {};
 
+  std::vector<int> _python_register_indices;
+
 private:
   std::string format_cgroup_file(pid_t pid, std::string_view path_to_proc);
 
@@ -44,6 +51,8 @@ class Process {
   pid_t _pid;
   CGroupId_t _cgroup_ns;
   ContainerId _container_id;
+
+  austin_handle_t _austin_handle = nullptr;
 };
 
 class ProcessHdr {
@@ -53,6 +62,8 @@ class ProcessHdr {
   const ContainerId &get_container_id(pid_t pid, bool force = false);
   void clear(pid_t pid) { _process_map.erase(pid); }
 
+  Process &get_process(pid_t pid);
+
 private:
   constexpr static auto k_nb_samples_container_id_lookup = 100;
   using ProcessMap = std::unordered_map<pid_t, Process>;

include/libaustin/libaustin.h

@@ -0,0 +1,60 @@
+// This file is part of "austin" which is released under GPL.
+//
+// See file LICENCE or go to http://www.gnu.org/licenses/ for full license
+// details.
+//
+// Austin is a Python frame stack sampler for CPython.
+//
+// Copyright (c) 2018 Gabriele N. Tornetta <[email protected]>.
+// All rights reserved.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef LIBAUSTIN_H
+#define LIBAUSTIN_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <sys/types.h>
+
+typedef void (*austin_callback_t)(pid_t, pid_t);
+typedef void *austin_handle_t;
+typedef struct {
+  uintptr_t key; // private
+  char *filename;
+  char *scope;
+  unsigned int line;
+} austin_frame_t;
+
+extern int austin_up();
+
+extern void austin_down();
+
+extern austin_handle_t austin_attach(pid_t);
+
+extern void austin_detach(austin_handle_t);
+
+extern int austin_sample(austin_handle_t, austin_callback_t);
+
+extern int austin_sample_thread(austin_handle_t, pid_t);
+
+extern austin_frame_t *austin_pop_frame();
+
+austin_frame_t *austin_read_frame(austin_handle_t, void *);
+
+#ifdef __cplusplus
+}
+#endif
+#endif // LIBAUSTIN_H

include/symbol.hpp

@@ -21,7 +21,7 @@ class Symbol {
   Symbol(std::string symname, std::string demangle_name, uint32_t lineno,
          std::string srcpath)
       : _symname(std::move(symname)), _demangle_name(std::move(demangle_name)),
-        _lineno(lineno), _srcpath(std::move(srcpath)) {}
+        _lineno(lineno), _srcpath(std::move(srcpath)), _is_python_frame(false) {}
 
   // OUTPUT OF ADDRINFO
   std::string _symname;
@@ -32,5 +32,7 @@ class Symbol {
   // OUTPUT OF LINE INFO
   uint32_t _lineno;
   std::string _srcpath;
+
+  bool _is_python_frame;
 };
 } // namespace ddprof

include/symbol_hdr.hpp

@@ -5,6 +5,7 @@
 
 #pragma once
 
+#include "austin_symbol_lookup.hpp"
 #include "base_frame_symbol_lookup.hpp"
 #include "common_mapinfo_lookup.hpp"
 #include "common_symbol_lookup.hpp"
@@ -42,6 +43,7 @@ struct SymbolHdr {
   ddprof::DsoSymbolLookup _dso_symbol_lookup;
   ddprof::DwflSymbolLookup _dwfl_symbol_lookup;
   ddprof::RuntimeSymbolLookup _runtime_symbol_lookup;
+  ddprof::AustinSymbolLookup _austin_symbol_lookup;
   // Symbol table (contains the references to strings)
   ddprof::SymbolTable _symbol_table;
 

include/unwind.hpp

@@ -16,7 +16,7 @@ void unwind_init(void);
 
 // Fill sample info to prepare for unwinding
 void unwind_init_sample(UnwindState *us, uint64_t *sample_regs,
-                        pid_t sample_pid, uint64_t sample_size_stack,
+                        pid_t sample_pid, pid_t sample_tid, uint64_t sample_size_stack,
                         char *sample_data_stack);
 
 // Main unwind API

include/unwind_state.hpp

@@ -16,6 +16,8 @@
 #include "symbol_hdr.hpp"
 #include "unwind_output.hpp"
 
+#include "libaustin.h"
+
 #include <sys/types.h>
 
 typedef struct Dwfl Dwfl;
@@ -52,6 +54,7 @@ struct UnwindState {
   ddprof::ProcessHdr process_hdr;
 
   pid_t pid;
+  pid_t tid;
   char *stack;
   size_t stack_sz;
 

src/austin_symbol_lookup.cc

@@ -0,0 +1,36 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#include <iterator>
+#include <limits>
+#include <string.h>
+#include <string>
+#include <vector>
+
+#include "austin_symbol_lookup.hpp"
+
+namespace ddprof {
+
+SymbolIdx_t AustinSymbolLookup::get_or_insert(austin_frame_t *frame,
+                                              SymbolTable &symbol_table) {
+  SymbolIdx_t idx = -1;
+  // reuse elements to avoid growing the table
+  if (!_free_list.empty()) {
+    auto el = _free_list.back();
+    _free_list.pop_back();
+    _austin_symbols.push_back(el);
+    idx = el;
+  }
+  if (idx == -1) {
+    idx = symbol_table.size();
+    symbol_table.push_back(Symbol());
+  }
+  std::string symname = std::string(frame->scope);
+  symbol_table[idx] =
+      Symbol(symname, symname, frame->line, std::string(frame->filename));
+  return idx;
+}
+
+} // namespace ddprof

src/ddprof_process.cc

@@ -8,6 +8,7 @@
 #include "ddres.hpp"
 #include "string_format.hpp"
 
+#include "libaustin.h"
 #include <charconv> // for std::from_chars
 #include <unistd.h>
 #include <vector>
@@ -20,6 +21,21 @@ std::string Process::format_cgroup_file(pid_t pid,
   return string_format("%s/proc/%d/cgroup", path_to_proc.data(), pid);
 }
 
+Process::~Process() {
+  if (_austin_handle) {
+    austin_detach(_austin_handle);
+  }
+}
+
+austin_handle_t Process::get_austin_handle(pid_t tid) {
+  if (!_austin_handle) {
+    LG_NTC("Attaching to PID %d/%d", _pid, tid);
+    _austin_handle = austin_attach(_pid);
+    LG_NTC("Result of attach %s", _austin_handle?"Success":"Failure");
+  }
+  return _austin_handle;
+}
+
 const ContainerId &Process::get_container_id(std::string_view path_to_proc) {
   if (!_container_id) {
     extract_container_id(format_cgroup_file(_pid, path_to_proc), _container_id);
@@ -74,6 +90,22 @@ DDRes Process::read_cgroup_ns(pid_t pid, std::string_view path_to_proc,
   return {};
 }
 
+Process &ProcessHdr::get_process(pid_t pid) {
+  auto it = _process_map.find(pid);
+  if (it == _process_map.end()) {
+    // new process, parse cgroup
+    auto pair = _process_map.try_emplace(pid, pid);
+    if (pair.second) {
+      it = pair.first;
+    } else {
+      // todo: probably not an exception
+      LG_WRN("[ProcessHdr] Unable to insert process element");
+      throw std::runtime_error("Unable to insert process element");
+    }
+  }
+  return it->second;
+}
+
 const ContainerId &ProcessHdr::get_container_id(pid_t pid, bool force) {
   // lookup cgroup
   static const ContainerId unknown_container_id =

src/ddprof_worker.cc

@@ -234,7 +234,7 @@ static DDRes ddprof_unwind_sample(DDProfContext &ctx, perf_event_sample *sample,
   ddprof_stats_add(STATS_UNWIND_AVG_STACK_SIZE, sample->size_stack, nullptr);
 
   // copy the sample context into the unwind structure
-  unwind_init_sample(us, sample->regs, sample->pid, sample->size_stack,
+  unwind_init_sample(us, sample->regs, sample->pid, sample->tid, sample->size_stack,
                      sample->data_stack);
 
   // If a sample has a PID, it has a TID.  Include it for downstream labels

src/dwfl_symbol_lookup.cc

@@ -134,6 +134,12 @@ SymbolIdx_t DwflSymbolLookup::insert(const DDProfMod &ddprof_mod,
     table.push_back(std::move(symbol));
 
     Symbol &sym_ref = table.back();
+    // slightly hacky way to detect python frame we should replace
+    if (sym_ref._symname.find("PyEval_EvalFrameDefault") != std::string::npos ||
+        sym_ref._symname.find("PyEval_EvalFrameEx") != std::string::npos) {
+      // flag this as something we can replace
+      sym_ref._is_python_frame = true;
+    }
     if (sym_ref._srcpath.empty()) {
       // override with info from dso (this slightly mixes mappings and sources)
       // But it helps a lot at Datadog (as mappings are ignored for now in UI)

src/pprof/ddprof_pprof.cc

@@ -268,8 +268,9 @@ DDRes pprof_aggregate(const UnwindOutput *uw_output,
   ddog_prof_Profile_AddResult add_res = ddog_prof_Profile_add(profile, sample);
   if (add_res.tag == DDOG_PROF_PROFILE_ADD_RESULT_ERR) {
     defer { ddog_Error_drop(&add_res.err); };
-    DDRES_RETURN_ERROR_LOG(DD_WHAT_PPROF, "Unable to add profile: %s",
-                           add_res.err.message.ptr);
+    LG_NTC("Unable to add profile - continue execution.");
+//    DDRES_RETURN_ERROR_LOG(DD_WHAT_PPROF, "Unable to add profile: %s",
+//                           add_res.err.message.ptr);
   }
   return ddres_init();
 }