From 655a7ccf9ce21095aa3dcd499b3c2ecce66334fc Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:00:27 +0200 Subject: [PATCH 01/21] feat(memtrack): capture allocation stacks in eBPF Copy the caller's user stack in chunks at allocator entry and fold an FNV-1a digest over it in the kernel. The digest rides on the allocation event as stack_hash; the copied bytes, a DWARF-numbered register snapshot and a frame-pointer walk are emitted once per distinct digest on a dedicated ring buffer, so unwinding and symbolication can happen offline. Capture stays off until userspace sets the rodata toggle, so allocator probes are unchanged by default. Refs COD-3222 --- crates/memtrack/src/ebpf/c/allocator.h | 25 ++- crates/memtrack/src/ebpf/c/event.h | 58 ++++- crates/memtrack/src/ebpf/c/main.bpf.c | 1 + .../memtrack/src/ebpf/c/stack_capture.bpf.h | 210 ++++++++++++++++++ .../memtrack/src/ebpf/c/utils/event_helpers.h | 21 +- .../memtrack/src/ebpf/c/utils/map_helpers.h | 8 + 6 files changed, 304 insertions(+), 19 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/stack_capture.bpf.h diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 9a4cc2387..8de96317f 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -9,6 +9,7 @@ BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ SEC(UPROBE_SEC) \ int uprobe_##name(struct pt_regs* ctx) { \ + stash_stack_hash(capture_stack(ctx)); \ return store_param(&name##_arg, arg_expr); \ } \ SEC(URETPROBE_SEC) \ @@ -17,6 +18,7 @@ if (!arg_ptr) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ __u64 ret_val = PT_REGS_RC(ctx); \ if (ret_val == 0) { \ return 0; \ @@ -32,6 +34,7 @@ if (arg0 == 0) { \ return 0; \ } \ + __u64 stack_hash = capture_stack(ctx); \ submit_block; \ } @@ -50,6 +53,8 @@ return 0; \ } \ \ + stash_stack_hash(capture_stack(ctx)); \ + \ struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr}; \ \ bpf_map_update_elem(&name##_args, &tid, &args, BPF_ANY); \ @@ -63,6 +68,7 @@ if (!args) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ \ struct name##_args_t a = *args; \ bpf_map_delete_elem(&name##_args, &tid); \ @@ -77,20 +83,22 @@ submit_block; \ } -UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), + { return submit_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); }) +UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0, stack_hash); }) UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx), - { return submit_calloc_event(arg0, ret_val); }) + { return submit_calloc_event(arg0, ret_val, stack_hash); }) UPROBE_ARGS_RET(realloc, PT_REGS_PARM2(ctx), PT_REGS_PARM1(ctx), - { return submit_realloc_event(arg1, ret_val, arg0); }) + { return submit_realloc_event(arg1, ret_val, arg0, stack_hash); }) UPROBE_ARG_RET(aligned_alloc, PT_REGS_PARM2(ctx), - { return submit_aligned_alloc_event(arg0, ret_val); }) + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), { return submit_aligned_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) /* * posix_memalign(void** memptr, size_t alignment, size_t size) @@ -115,6 +123,8 @@ int uprobe_posix_memalign(struct pt_regs* ctx) { return 0; } + stash_stack_hash(capture_stack(ctx)); + struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)}; bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY); return 0; @@ -127,6 +137,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { if (!args) { return 0; } + __u64 stack_hash = take_stack_hash(); struct posix_memalign_args_t a = *args; bpf_map_delete_elem(&posix_memalign_args, &tid); @@ -140,7 +151,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { return 0; } - return submit_aligned_alloc_event(a.size, addr); + return submit_aligned_alloc_event(a.size, addr, stack_hash); } struct mmap_args { diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index bf0677c93..7e10a7756 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -15,6 +15,49 @@ #define EVENT_TYPE_RSS 12 #define EVENT_TYPE_RMAP 13 +/* Largest user-stack copy one definition can carry. Every capture reserves + * header + budget in the ring up front, so this bounds ring space held per + * in-flight capture, per-capture copy cost, and the verifier work per load + * (which scales with the frozen budget). The kernel itself allows records up + * to the ring size; this is a policy cap. */ +#define MEMTRACK_MAX_STACK_COPY (32 * 1024) + +/* Registers, indexed by the capturing architecture's DWARF register number + * (x86_64: 0=rax .. 7=rsp, 8..15=r8-r15, 16=rip; aarch64: 0..30=x0-x30, + * 31=sp, 32=pc). Slots the architecture does not define stay zero. An offline + * DWARF unwinder needs the callee-saved ones to evaluate CFA rules, not just + * ip/sp/bp. */ +#define MEMTRACK_STACK_REGS 33 + +/* Counter slots in the stack_counters array map. */ +#define MEMTRACK_STACK_COUNTER_COPY_FAILED 0 +#define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1 +/* bpf_get_stackid() has several negative outcomes (no user callchain, + * hash-bucket collision, or no free bucket), so this counts only missing ids. */ +#define MEMTRACK_STACK_COUNTER_STACKID_FAILED 2 +#define MEMTRACK_STACK_COUNTER_TRUNCATED 3 +#define MEMTRACK_STACK_COUNTER_RING_FULL 4 +#define MEMTRACK_STACK_COUNTER_COUNT 5 + +struct stack_regs { + uint64_t reg[MEMTRACK_STACK_REGS]; +}; + +/* Head of a stack record; `copy_len` raw stack bytes read upwards from `sp` + * follow it. */ +struct stack_header { + uint64_t hash; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ + uint64_t sp; /* user stack pointer the copy starts at */ + uint32_t pid; + uint32_t tid; + uint32_t copy_len; + uint8_t truncated; /* the copy hit the size cap */ + uint8_t _pad[3]; + struct stack_regs regs; +}; + /* Common header shared by all event types */ struct event_header { uint8_t event_type; /* See EVENT_TYPE_* constants above */ @@ -29,20 +72,23 @@ struct event { union { /* Allocation events (malloc, calloc, aligned_alloc) */ struct { - uint64_t addr; /* address returned */ - uint64_t size; /* size requested */ + uint64_t addr; /* address returned */ + uint64_t size; /* size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } alloc; /* Deallocation event (free) */ struct { - uint64_t addr; /* address to free */ + uint64_t addr; /* address to free */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } free; /* Reallocation event - includes both old and new addresses */ struct { - uint64_t old_addr; /* previous address (can be NULL) */ - uint64_t new_addr; /* new address returned */ - uint64_t size; /* new size requested */ + uint64_t old_addr; /* previous address (can be NULL) */ + uint64_t new_addr; /* new address returned */ + uint64_t size; /* new size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } realloc; /* Memory mapping events (mmap, munmap, brk) */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 5a8d6ff07..b405f572b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -11,6 +11,7 @@ #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" +#include "stack_capture.bpf.h" #include "utils/event_helpers.h" #include "utils/folio.h" #include "utils/map_helpers.h" diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h new file mode 100644 index 000000000..2519767b5 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -0,0 +1,210 @@ +#ifndef __STACK_CAPTURE_BPF_H__ +#define __STACK_CAPTURE_BPF_H__ + +#include "event.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* Emit raw stack bytes and registers once per hash for offline DWARF unwinding. + * Allocation events carry the hash; stackid provides the frame-pointer fallback. + * Hashes may repeat after stack data changes or LRU eviction. + */ + +const volatile __u8 capture_stacks_enabled = 0; +const volatile __u32 stack_copy_budget = 4096; + +#define STACK_TRACE_MAX_DEPTH 127 +#define STACK_COPY_CHUNK 512 +#define FNV64_OFFSET 0xcbf29ce484222325ULL +#define FNV64_PRIME 0x00000100000001b3ULL + +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(max_entries, 16384); + __type(key, __u32); + __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); +} stack_traces SEC(".maps"); + +/* A separate ring keeps allocation events fixed-size. */ +BPF_RINGBUF(stacks, 512 * 1024 * 1024); +BPF_LRU_HASH_MAP(seen_stack_hashes, __u64, __u8, 262144); +BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); +BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); + +static __always_inline void bump_stack_counter(__u32 index) { + __u64* slot = bpf_map_lookup_elem(&stack_counters, &index); + if (slot) { + __sync_fetch_and_add(slot, 1); + } +} + +/* 4-lane FNV-1a over one STACK_COPY_CHUNK worth of 8-byte words. Fixed-size, + * unrolled so the verifier sees a bounded loop. */ +static __always_inline void fnv64_hash_chunk(__u64 lanes[4], const __u64* words) { +#pragma unroll + for (__u32 i = 0; i < STACK_COPY_CHUNK / 8; i += 4) { + lanes[0] = (lanes[0] ^ words[i]) * FNV64_PRIME; + lanes[1] = (lanes[1] ^ words[i + 1]) * FNV64_PRIME; + lanes[2] = (lanes[2] ^ words[i + 2]) * FNV64_PRIME; + lanes[3] = (lanes[3] ^ words[i + 3]) * FNV64_PRIME; + } +} + +#if defined(__TARGET_ARCH_x86) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + out->reg[0] = ctx->ax; + out->reg[1] = ctx->dx; + out->reg[2] = ctx->cx; + out->reg[3] = ctx->bx; + out->reg[4] = ctx->si; + out->reg[5] = ctx->di; + out->reg[6] = ctx->bp; + out->reg[7] = ctx->sp; + out->reg[8] = ctx->r8; + out->reg[9] = ctx->r9; + out->reg[10] = ctx->r10; + out->reg[11] = ctx->r11; + out->reg[12] = ctx->r12; + out->reg[13] = ctx->r13; + out->reg[14] = ctx->r14; + out->reg[15] = ctx->r15; + out->reg[16] = ctx->ip; +} +#elif defined(__TARGET_ARCH_arm64) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + struct user_pt_regs* uregs = (struct user_pt_regs*)ctx; +#pragma unroll + for (int i = 0; i < 31; i++) { + out->reg[i] = uregs->regs[i]; + } + out->reg[31] = uregs->sp; + out->reg[32] = uregs->pc; +} +#else +#error "stack capture needs a DWARF register mapping for this architecture" +#endif + +static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { + void* slot = bpf_ringbuf_reserve(&stacks, sizeof(struct stack_header) + stack_copy_budget, 0); + if (!slot) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_RING_FULL); + return 0; + } + + __u64 sp = PT_REGS_SP(ctx); + __u8* payload = (__u8*)slot + sizeof(struct stack_header); + __u64 lanes[4] = { + FNV64_OFFSET ^ 0, + FNV64_OFFSET ^ 1, + FNV64_OFFSET ^ 2, + FNV64_OFFSET ^ 3, + }; + __u32 got = 0; + + /* Chunked reads stop at the first unreadable stack region. + * Loop bound is checked against stack_copy_budget (a frozen rodata constant) + * so every slot access is provably in range. */ +#pragma clang loop unroll(disable) + for (__u32 off = 0; off + STACK_COPY_CHUNK <= stack_copy_budget; off += STACK_COPY_CHUNK) { + if (bpf_probe_read_user(payload + off, STACK_COPY_CHUNK, (void*)(sp + off)) != 0) { + break; + } + + fnv64_hash_chunk(lanes, (const __u64*)(payload + off)); + got = off + STACK_COPY_CHUNK; + } + + if (got == 0) { + bpf_ringbuf_discard(slot, 0); + bump_stack_counter(MEMTRACK_STACK_COUNTER_COPY_FAILED); + return 0; + } + + __u8 truncated = got >= stack_copy_budget; + if (truncated) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_TRUNCATED); + } + + __u64 hash = + (((lanes[0] * FNV64_PRIME) ^ lanes[1]) * FNV64_PRIME ^ lanes[2]) * FNV64_PRIME ^ lanes[3]; + + /* Length distinguishes a full copy from the same bytes as a truncated prefix. + * Zero is reserved for allocation events without a stack. */ + hash = (hash ^ got) * FNV64_PRIME; + if (hash == 0) { + hash = FNV64_OFFSET; + } + + __u8 marker = 1; + long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); + if (gate_result == -17) { /* -EEXIST */ + bpf_ringbuf_discard(slot, 0); + return hash; + } + if (gate_result != 0) { + /* Re-emit when deduplication is full so the hash remains resolvable. */ + bump_stack_counter(MEMTRACK_STACK_COUNTER_HASH_MAP_FULL); + } + + __s64 stackid = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK); + if (stackid < 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_STACKID_FAILED); + } + + struct stack_header* header = (struct stack_header*)slot; + header->hash = hash; + header->timestamp = bpf_ktime_get_ns(); + header->stackid = stackid; + header->sp = sp; + header->pid = ids.tgid; + header->tid = ids.tid; + header->copy_len = got; + header->truncated = truncated; + header->_pad[0] = 0; + header->_pad[1] = 0; + header->_pad[2] = 0; + fill_stack_regs(&header->regs, ctx); + + bpf_ringbuf_submit(slot, 0); + return hash; +} + +static __always_inline __u64 capture_stack(struct pt_regs* ctx) { + if (!capture_stacks_enabled || !is_enabled()) { + return 0; + } + + struct task_ids ids = current_task_ids(); + if (!is_tracked(ids.tgid)) { + return 0; + } + + return capture_stack_inner(ctx, ids); +} + +static __always_inline void stash_stack_hash(__u64 hash) { + if (hash == 0) { + return; + } + + __u64 tid = current_tid(); + bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); +} + +static __always_inline __u64 take_stack_hash(void) { + if (!capture_stacks_enabled) { + return 0; + } + + __u64 tid = current_tid(); + __u64* hash = bpf_map_lookup_elem(&pending_stack_hash, &tid); + if (!hash) { + return 0; + } + + __u64 value = *hash; + bpf_map_delete_elem(&pending_stack_hash, &tid); + return value; +} + +#endif /* __STACK_CAPTURE_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index ca5969a9a..ca53593d2 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -2,6 +2,7 @@ #define __EVENT_HELPERS_H__ #include "../event.h" +#include "../stack_capture.bpf.h" #include "map_helpers.h" #include "process_tracking.h" @@ -88,36 +89,44 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(owner.tgid, evt_type, fill_data); \ } -static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_calloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); +static __always_inline int submit_free_event(__u64 addr, __u64 stack_hash) { + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { + e->data.free.addr = addr; + e->data.free.stack_hash = stack_hash; + }); } -static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { +static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size, + __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; + e->data.realloc.stack_hash = stack_hash; }); } diff --git a/crates/memtrack/src/ebpf/c/utils/map_helpers.h b/crates/memtrack/src/ebpf/c/utils/map_helpers.h index 484fe9703..69422cdf9 100644 --- a/crates/memtrack/src/ebpf/c/utils/map_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/map_helpers.h @@ -9,6 +9,14 @@ __type(value, value_type); \ } name SEC(".maps") +#define BPF_LRU_HASH_MAP(name, key_type, value_type, max_ents) \ + struct { \ + __uint(type, BPF_MAP_TYPE_LRU_HASH); \ + __uint(max_entries, max_ents); \ + __type(key, key_type); \ + __type(value, value_type); \ + } name SEC(".maps") + #define BPF_ARRAY_MAP(name, value_type, max_ents) \ struct { \ __uint(type, BPF_MAP_TYPE_ARRAY); \ From f9a09795aa1c901aaec42bbc103445a178d7671f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:00:53 +0200 Subject: [PATCH 02/21] feat(memtrack): add userspace stack-capture module Add the userspace half of allocation stack capture: env-driven configuration, stack-definition ring parsing, loss counters, per-pid module mapping tracking, a folding recorder that deduplicates definitions and counts occurrences, and the report it produces. Nothing constructs these yet; the tracker wiring follows. Refs COD-3222 --- Cargo.lock | 11 ++ crates/memtrack/src/ebpf/events.rs | 166 +++++++++++++++++- crates/memtrack/src/ebpf/mod.rs | 3 + crates/memtrack/src/ebpf/stacks.rs | 35 ++++ crates/memtrack/tests/c_tests.rs | 2 +- crates/memtrack/tests/dlopen_tests.rs | 11 +- crates/memtrack/tests/shared.rs | 7 +- crates/runner-shared/Cargo.toml | 5 + .../runner-shared/benches/memtrack_writer.rs | 160 ++++++++++++++--- .../src/artifacts/memtrack/mod.rs | 93 ++++++++-- .../src/artifacts/memtrack/pipeline.rs | 5 +- 11 files changed, 446 insertions(+), 52 deletions(-) create mode 100644 crates/memtrack/src/ebpf/stacks.rs diff --git a/Cargo.lock b/Cargo.lock index 1acec08f4..8e555476f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3616,6 +3616,7 @@ dependencies = [ "rmp", "rmp-serde", "serde", + "serde_bytes", "serde_json", "zstd", ] @@ -4062,6 +4063,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 4ed422a58..5a8abfce0 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -1,4 +1,6 @@ -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use crate::prelude::*; +use libbpf_rs::MapCore; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, StackRecord}; // Include the bindings for event.h pub mod bindings { @@ -34,13 +36,20 @@ pub fn parse_event(data: &[u8]) -> Option { event.data.alloc.addr, MemtrackEventKind::Malloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, + }, + ), + EVENT_TYPE_FREE => ( + event.data.free.addr, + MemtrackEventKind::Free { + stack_hash: event.data.free.stack_hash, }, ), - EVENT_TYPE_FREE => (event.data.free.addr, MemtrackEventKind::Free), EVENT_TYPE_CALLOC => ( event.data.alloc.addr, MemtrackEventKind::Calloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_REALLOC => ( @@ -48,12 +57,14 @@ pub fn parse_event(data: &[u8]) -> Option { MemtrackEventKind::Realloc { old_addr: Some(event.data.realloc.old_addr), size: event.data.realloc.size, + stack_hash: event.data.realloc.stack_hash, }, ), EVENT_TYPE_ALIGNED_ALLOC => ( event.data.alloc.addr, MemtrackEventKind::AlignedAlloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_MMAP => ( @@ -111,6 +122,74 @@ pub fn parse_event(data: &[u8]) -> Option { }) } +/// Decode one stack record from the ring buffer, returning it alongside the +/// `bpf_get_stackid()` result its frame-pointer chain is stored under. +pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { + let header_len = std::mem::size_of::(); + // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. + let header: stack_header = if data.len() >= header_len { + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } + } else { + warn!( + "malformed stack record: {} bytes, need at least {header_len}", + data.len() + ); + return None; + }; + + let record_len = header_len + header.copy_len as usize; + if data.len() < record_len { + warn!( + "malformed stack record: {} bytes, need {record_len}", + data.len() + ); + return None; + } + + let event = MemtrackEvent { + pid: header.pid as i32, + tid: header.tid as i32, + timestamp: header.timestamp, + addr: 0, + kind: MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: header.hash, + sp: header.sp, + regs: header.regs.reg.to_vec(), + bytes: data[header_len..record_len].to_vec(), + fp_chain: Vec::new(), + truncated: header.truncated != 0, + }), + }, + }; + + Some((event, header.stackid)) +} + +/// The frame-pointer walk recorded under `stackid`, innermost frame first. +/// Best effort: a missing chain costs the fallback for one stack, not the run. +pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() +} + /// A request from the exec-mapping watcher to attach allocator probes. #[derive(Debug, Clone, Copy)] pub struct AttachRequest { @@ -157,6 +236,7 @@ mod tests { event.data.realloc.old_addr = 0x1000; event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; + event.data.realloc.stack_hash = 0xbeef; let bytes = event_bytes(&event); @@ -168,9 +248,14 @@ mod tests { assert_eq!(parsed.addr, 0x2000); match parsed.kind { - MemtrackEventKind::Realloc { old_addr, size } => { + MemtrackEventKind::Realloc { + old_addr, + size, + stack_hash, + } => { assert_eq!(old_addr, Some(0x1000)); assert_eq!(size, 256); + assert_eq!(stack_hash, 0xbeef); } _ => panic!("Expected Realloc event kind"), } @@ -186,6 +271,7 @@ mod tests { event.header.tid = 2000; event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; + event.data.alloc.stack_hash = 0x1234; let bytes = event_bytes(&event); @@ -197,8 +283,9 @@ mod tests { assert_eq!(parsed.addr, 0x1000); match parsed.kind { - MemtrackEventKind::Malloc { size } => { + MemtrackEventKind::Malloc { size, stack_hash } => { assert_eq!(size, 128); + assert_eq!(stack_hash, 0x1234); } _ => panic!("Expected Malloc event kind"), } @@ -277,3 +364,74 @@ mod tests { } } } + +#[cfg(test)] +mod stack_tests { + use super::*; + use crate::ebpf::events::bindings::stack_regs; + + fn encode(header: stack_header, payload: &[u8]) -> Vec { + // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const stack_header).cast::(), + std::mem::size_of::(), + ) + }; + let mut data = header_bytes.to_vec(); + data.extend_from_slice(payload); + data + } + + fn header(copy_len: u32) -> stack_header { + stack_header { + hash: 0x0123_4567_89ab_cdef, + timestamp: 987_654_321, + stackid: -17, + sp: 0x7fff_1234_5000, + pid: 41, + tid: 42, + copy_len, + truncated: 1, + _pad: [0; 3], + regs: stack_regs { + reg: std::array::from_fn(|index| 0x1000 + index as u64), + }, + } + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let header = header(5); + let payload = [1, 2, 3, 4, 5]; + + let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); + assert_eq!(event.pid, 41); + assert_eq!(event.tid, 42); + assert_eq!(event.timestamp, 987_654_321); + assert_eq!(event.addr, 0); + assert_eq!(stackid, -17); + + let MemtrackEventKind::Stack { record } = event.kind else { + panic!("expected Stack event"); + }; + + assert_eq!(record.hash, header.hash); + assert_eq!(record.sp, header.sp); + assert_eq!(record.regs, header.regs.reg.to_vec()); + assert_eq!(record.bytes, payload); + assert!(record.fp_chain.is_empty()); + assert!(record.truncated); + } + + #[test] + fn truncated_buffer_returns_none() { + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack(&data).is_none()); + } + + #[test] + fn missing_payload_returns_none() { + assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 2aa96549d..d964ebed5 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -4,9 +4,12 @@ mod memtrack; pub(crate) mod poller; mod proc_fs; mod spawn; +mod stacks; mod tracker; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; +pub use stacks::config::{DEFAULT_STACK_COPY_SIZE, clamp_copy_size}; +pub use stacks::counters::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/stacks.rs b/crates/memtrack/src/ebpf/stacks.rs new file mode 100644 index 000000000..88605294b --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks.rs @@ -0,0 +1,35 @@ +use crate::ebpf::events::bindings::*; +use crate::prelude::*; + +#[derive(Debug, Clone, Copy, Default, serde::Serialize)] +pub struct StackCaptureStats { + pub copy_failed: u64, + pub hash_map_full: u64, + pub stackid_failed: u64, + pub truncated: u64, + pub ring_full: u64, +} + +impl StackCaptureStats { + pub fn read(map: &impl libbpf_rs::MapCore) -> Result { + Ok(Self { + copy_failed: slot(map, MEMTRACK_STACK_COUNTER_COPY_FAILED)?, + hash_map_full: slot(map, MEMTRACK_STACK_COUNTER_HASH_MAP_FULL)?, + stackid_failed: slot(map, MEMTRACK_STACK_COUNTER_STACKID_FAILED)?, + truncated: slot(map, MEMTRACK_STACK_COUNTER_TRUNCATED)?, + ring_full: slot(map, MEMTRACK_STACK_COUNTER_RING_FULL)?, + }) + } +} + +fn slot(map: &impl libbpf_rs::MapCore, index: u32) -> Result { + let value = map + .lookup(&index.to_ne_bytes(), libbpf_rs::MapFlags::ANY) + .with_context(|| format!("failed to read stack counter {index}"))? + .ok_or_else(|| anyhow!("stack counter slot {index} missing"))?; + let bytes: [u8; 8] = value + .as_slice() + .try_into() + .map_err(|_| anyhow!("stack counter {index} has unexpected size"))?; + Ok(u64::from_ne_bytes(bytes)) +} diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index db5fe6439..2d33dccf0 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -110,7 +110,7 @@ fn test_track_allocators_disabled_skips_allocations() -> Result<(), Box Result<(), Box> { let malloc_addrs: HashSet = events .iter() .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size: 4242 } => Some(e.addr), + MemtrackEventKind::Malloc { size: 4242, .. } => Some(e.addr), _ => None, }) .collect(); let malloc_count = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let free_count = events .iter() .filter(|e| { - matches!(e.kind, MemtrackEventKind::Free) && malloc_addrs.contains(&e.addr) + matches!(e.kind, MemtrackEventKind::Free { .. }) + && malloc_addrs.contains(&e.addr) }) .count(); @@ -125,11 +126,11 @@ fn test_thread_dlopen() -> Result<(), Box> { |events| { let m4242 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let m4243 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243, .. })) .count(); assert_eq!(m4242, 100, "expected 100 mi_malloc(4242) events"); diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 0d27b6a46..7d51c7025 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -31,7 +31,7 @@ macro_rules! assert_events_snapshot { matches!( e.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } @@ -108,7 +108,7 @@ pub fn between_markers(events: &[Event]) -> Vec { const MARKER: u64 = 0xC0D5_9EED; let is_marker = - |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size } if size == MARKER); + |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size, .. } if size == MARKER); events .iter() @@ -124,6 +124,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + ) }) .sorted_by_key(|e| e.timestamp) @@ -281,7 +282,7 @@ fn event_profile(events: &[Event]) -> EventProfile { if !matches!( event.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } diff --git a/crates/runner-shared/Cargo.toml b/crates/runner-shared/Cargo.toml index 8b8f6ab97..aee1707f0 100644 --- a/crates/runner-shared/Cargo.toml +++ b/crates/runner-shared/Cargo.toml @@ -4,9 +4,14 @@ publish = false version = "0.1.0" edition = "2024" +# Set by `cargo codspeed build` for the whole build; benches branch on it. +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(codspeed)'] } + [dependencies] anyhow = { workspace = true } serde = { workspace = true } +serde_bytes = "0.11" serde_json = { workspace = true } # Pinned to 1.x: 2.0 changes the wire format and serde integration bincode = "1.3" diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index a6c610e8e..1479fe96c 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -1,7 +1,22 @@ use divan::Bencher; +use divan::counter::{BytesCount, ItemsCount}; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, MemtrackWriter, encode_events}; +use runner_shared::artifacts::{ + MemtrackEvent, MemtrackEventKind, MemtrackWriter, StackRecord, encode_events, +}; + +/// Reports allocation counts and bytes next to the timings for local runs. Only +/// tallies the thread running the benchmark, so the parallel encoder's +/// per-worker allocations show up in the single-threaded writer benches +/// instead. +/// +/// Left out of CodSpeed builds: wrapping the allocator costs ~15% on +/// allocation-heavy benchmarks, and the memory instrument reports allocations +/// there anyway. +#[cfg(not(codspeed))] +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); fn main() { divan::main(); @@ -14,14 +29,24 @@ fn generate_events(n: usize) -> Vec { for _ in 0..n { let size = rng.gen_range(8..8192); let kind = match rng.gen_range(0..10) { - 0 => MemtrackEventKind::Malloc { size }, - 1 => MemtrackEventKind::Free, + 0 => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, + 1 => MemtrackEventKind::Free { stack_hash: 0 }, 2 => MemtrackEventKind::Realloc { old_addr: Some(rng.r#gen()), size, + stack_hash: 0, + }, + 3 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, + 4 => MemtrackEventKind::AlignedAlloc { + size, + stack_hash: 0, }, - 3 => MemtrackEventKind::Calloc { size }, - 4 => MemtrackEventKind::AlignedAlloc { size }, 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, @@ -48,18 +73,25 @@ fn generate_events(n: usize) -> Vec { events } -#[divan::bench(args = [10_000, 100_000, 500_000, 1_000_000])] +/// Throughput of the single-threaded writer path: one zstd frame, no pool. +/// This is the per-worker ceiling the parallel encoder scales from. +#[divan::bench(args = [10_000, 100_000], max_time = 5.0)] fn write_events(bencher: Bencher, n: usize) { let events = generate_events(n); + let artifact_bytes = write_frame(&events).len(); + + bencher + .counter(ItemsCount::new(n)) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| write_frame(&events)); +} - bencher.bench_local(|| { - let mut output = Vec::new(); - let mut writer = MemtrackWriter::new(&mut output).unwrap(); - for event in &events { - writer.write_event(event).unwrap(); - } - writer.finish().unwrap(); - }); +fn write_frame(events: &[MemtrackEvent]) -> Vec { + let mut writer = MemtrackWriter::new(Vec::new()).unwrap(); + for event in events { + writer.write_event(event).unwrap(); + } + writer.finish().unwrap() } fn generate_realistic_events(n: usize) -> Vec { @@ -90,12 +122,18 @@ fn generate_realistic_events(n: usize) -> Vec { addr }); let kind = match rng.gen_range(0..20) { - 0 => MemtrackEventKind::Calloc { size }, + 0 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, 1 => MemtrackEventKind::Mmap { size }, - _ => MemtrackEventKind::Malloc { size }, + _ => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, }; - if let MemtrackEventKind::Mmap { size } = kind { - live_mmap.push((addr, size)); + if let MemtrackEventKind::Mmap { size } = &kind { + live_mmap.push((addr, *size)); } else { live_heap.push(addr); } @@ -105,7 +143,7 @@ fn generate_realistic_events(n: usize) -> Vec { if idx < live_heap.len() { let addr = live_heap.swap_remove(idx); free_list.push(addr); - (addr, MemtrackEventKind::Free) + (addr, MemtrackEventKind::Free { stack_hash: 0 }) } else { let (addr, size) = live_mmap.swap_remove(idx - live_heap.len()); free_list.push(addr); @@ -127,6 +165,7 @@ fn generate_realistic_events(n: usize) -> Vec { MemtrackEventKind::Realloc { old_addr: Some(old_addr), size, + stack_hash: 0, }, ) }; @@ -142,15 +181,84 @@ fn generate_realistic_events(n: usize) -> Vec { events } -const REALISTIC_EVENTS: usize = 1_000_000; +/// One event per frame slot of a full window, so every worker count from 1 to +/// `WINDOW_FRAMES` has a frame to take. Sizing below this hides pool scaling: +/// the encoder can only parallelize across whole frames. +const REALISTIC_EVENTS: usize = 16 * 64 * 1024; -#[divan::bench(args = [16, 8, 4], max_time = 10.0)] +/// Throughput of the artifact encoder over a realistic allocation mix, as a +/// function of the worker pool size. +#[divan::bench(args = [1, 2, 4, 8, 16], max_time = 10.0)] fn encode_events_realistic(bencher: Bencher, n_workers: usize) { let events = generate_realistic_events(REALISTIC_EVENTS); + let artifact_bytes = encode(&events, n_workers).len(); - bencher.bench_local(|| { - let mut output = Vec::new(); - encode_events(events.iter().copied(), &mut output, n_workers).unwrap(); - output - }); + bencher + .counter(ItemsCount::new(events.len())) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| encode(&events, n_workers)); +} + +/// Single-frame throughput on captured stacks: each `Stack` event carries a +/// register set and a raw stack copy, so payloads are orders of magnitude +/// larger than an allocation record and the byte rate is what matters. +#[divan::bench(max_time = 10.0)] +fn write_stack_events(bencher: Bencher) { + let events = generate_stack_events(16 * 1024); + let artifact_bytes = write_frame(&events).len(); + + bencher + .counter(ItemsCount::new(events.len())) + .counter(BytesCount::new(artifact_bytes)) + .bench_local(|| write_frame(&events)); +} + +fn encode(events: &[MemtrackEvent], n_workers: usize) -> Vec { + let mut output = Vec::new(); + encode_events(events.iter().cloned(), &mut output, n_workers).unwrap(); + output +} + +/// A stack-capture heavy stream: one `Stack` record per allocation, sized like +/// the kernel's stack copies (2 KiB payload, x86_64 register set). +fn generate_stack_events(n: usize) -> Vec { + const STACK_BYTES: usize = 2048; + let mut rng = StdRng::seed_from_u64(7); + let mut events = Vec::with_capacity(n * 2); + + while events.len() < n * 2 { + let hash: u64 = rng.r#gen(); + let record = StackRecord { + hash, + sp: 0x7fff_0000_0000 | (rng.gen_range(0..1u64 << 20) << 4), + regs: (0..33).map(|_| rng.r#gen()).collect(), + bytes: (0..STACK_BYTES).map(|_| rng.r#gen()).collect(), + fp_chain: (0..16).map(|_| rng.r#gen()).collect(), + truncated: false, + }; + let addr: u64 = rng.r#gen(); + let timestamp: u64 = rng.r#gen(); + + events.push(MemtrackEvent { + pid: 4242, + tid: 4242, + timestamp, + addr, + kind: MemtrackEventKind::Stack { + record: Box::new(record), + }, + }); + events.push(MemtrackEvent { + pid: 4242, + tid: 4242, + timestamp: timestamp + 1, + addr, + kind: MemtrackEventKind::Malloc { + size: rng.gen_range(8..8192), + stack_hash: hash, + }, + }); + } + + events } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index b082a7c67..433aab5a5 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -41,7 +41,7 @@ impl MemtrackArtifact { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct MemtrackEvent { pub pid: pid_t, pub tid: pid_t, @@ -51,23 +51,34 @@ pub struct MemtrackEvent { pub kind: MemtrackEventKind, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] pub enum MemtrackEventKind { Malloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, + }, + Free { + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, - Free, Realloc { #[serde(default, skip_serializing_if = "Option::is_none")] old_addr: Option, size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Calloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, AlignedAlloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Mmap { size: u64, @@ -91,6 +102,31 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + + Stack { + #[serde(flatten)] + record: Box, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StackRecord { + pub hash: u64, + /// User stack pointer the copy starts at. + pub sp: u64, + /// Registers by DWARF number for the capturing architecture; 33 entries on x86_64. + pub regs: Vec, + /// Raw stack bytes read upward from `sp`. + #[serde(with = "serde_bytes")] + pub bytes: Vec, + /// In-kernel frame-pointer walk, innermost first; empty when unavailable. + pub fp_chain: Vec, + /// The copy filled its budget, so stack above it was not captured. + pub truncated: bool, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 } pub struct MemtrackEventStream { @@ -120,14 +156,17 @@ mod tests { tid: 11, timestamp: 100, addr: 0x10, - kind: MemtrackEventKind::Malloc { size: 64 }, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, }, MemtrackEvent { pid: 1, tid: 12, timestamp: 200, addr: 0x20, - kind: MemtrackEventKind::Free, + kind: MemtrackEventKind::Free { stack_hash: 0 }, }, MemtrackEvent { pid: 1, @@ -167,21 +206,47 @@ mod tests { } let kinds = [ - MemtrackEventKind::Malloc { size: 7 }, - MemtrackEventKind::Free, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0, + }, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0xCAFE_BABE, + }, + MemtrackEventKind::Free { stack_hash: 0 }, + MemtrackEventKind::Free { stack_hash: 0xFEED }, MemtrackEventKind::Realloc { old_addr: Some(0x1000), size: 42, + stack_hash: 0, }, MemtrackEventKind::Realloc { old_addr: None, size: 42, + stack_hash: 0x1234, + }, + MemtrackEventKind::Calloc { + size: 9, + stack_hash: 0, + }, + MemtrackEventKind::AlignedAlloc { + size: 9, + stack_hash: 0, }, - MemtrackEventKind::Calloc { size: 9 }, - MemtrackEventKind::AlignedAlloc { size: 9 }, MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: 0xDEAD_BEEF, + sp: 0x7FFF_0000, + regs: vec![0; 33], + bytes: vec![1, 2, 3, 4], + fp_chain: vec![0x1000, 0x2000], + truncated: false, + }), + }, ]; for kind in kinds { @@ -190,7 +255,7 @@ mod tests { tid: 42, timestamp: 0xDEAD, addr: 0xBEEF, - kind, + kind: kind.clone(), }; let shadow = Shadow { pid: -7, @@ -215,7 +280,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect(); @@ -265,7 +333,8 @@ mod tests { event.kind, MemtrackEventKind::Realloc { old_addr: None, - size: 42 + size: 42, + stack_hash: 0, } )); diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index c47b3aed9..8cac46f05 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -94,7 +94,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect() } From aceaf717c36d940b7794cac39a4826e6987ff20f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 11:33:08 +0200 Subject: [PATCH 03/21] feat(memtrack): enable stack capture through the tracker Wire the capture rodata and map sizing into skeleton load, poll the stack-definition ring alongside the event ring, and expose the loss counters and frame-pointer chains. The attach worker snapshots module mappings while it holds a process stopped, which is the only point they are guaranteed readable. Guard the lifecycle: finishing with a live session would block forever on the recorder, and a second spawn would leave the capture rings undrained, so both now fail with a descriptive error. With capture disabled the ring buffer and frame-pointer map shrink to the allocator minimum rather than reserving tens of MiB. Refs COD-3222 --- crates/memtrack/src/ebpf/memtrack/maps.rs | 5 +++ crates/memtrack/src/ebpf/memtrack/mod.rs | 49 +++++++++++++++++++++++ crates/memtrack/src/ebpf/tracker.rs | 39 ++++++++++++++++-- crates/memtrack/src/main.rs | 7 ++++ crates/memtrack/src/session.rs | 3 ++ 5 files changed, 100 insertions(+), 3 deletions(-) diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index c7376d463..994650d53 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -1,4 +1,5 @@ use super::MemtrackBpf; +use crate::ebpf::stacks::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; @@ -68,6 +69,10 @@ impl MemtrackBpf { ) } + pub fn stack_capture_stats(&self) -> Result { + StackCaptureStats::read(with_skel!(self, skel => &skel.maps.stack_counters)) + } + pub fn ownership_maps(&self) -> Result { let owner_by_mm = entries(with_skel!(self, skel => &skel.maps.owner_by_mm))?; let mm_by_pid = entries(with_skel!(self, skel => &skel.maps.mm_by_pid))?; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 2ec1f04f5..cf3014608 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -27,6 +27,7 @@ pub use rmap::RmapSupport; use crate::bpf_token::has_delegated_bpf_token; use crate::ebpf::TrackerOptions; +use crate::ebpf::stacks::config::clamp_copy_size; /// Which attach mechanism a loaded skeleton uses for its uprobes. See /// `src/ebpf/c/utils/variant.h` for why only one of them is delegatable. @@ -125,6 +126,8 @@ pub struct MemtrackBpf { impl MemtrackBpf { /// Load the skeleton, defaulting to the variant a BPF token is available for. + /// + /// `options.stack_copy_size` turns on allocation stack capture. pub fn load(options: TrackerOptions) -> Result { let variant = options.variant.unwrap_or_else(|| { if has_delegated_bpf_token() { @@ -134,6 +137,7 @@ impl MemtrackBpf { } }); let physical = options.physical; + let stack_copy_size = options.stack_copy_size.map(clamp_copy_size); crate::kernel::KernelBtf::ensure_available()?; let page_shift = page_shift()?; @@ -162,6 +166,19 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } + if let Some(copy_size) = stack_copy_size { + rodata.capture_stacks_enabled = 1; + rodata.stack_copy_size = copy_size; + } + } + + // Avoid reserving the stack maps when capture is disabled. A + // ring buffer's size must stay a power-of-two page count. + if stack_copy_size.is_none() { + open_skel.maps.stacks.set_max_entries(4096)?; + open_skel.maps.stack_traces.set_max_entries(1)?; + open_skel.maps.seen_stack_hashes.set_max_entries(1)?; + open_skel.maps.pending_stack_hash.set_max_entries(1)?; } // Autoload is decided before load(), so missing fentry targets must be off here. @@ -228,6 +245,38 @@ impl MemtrackBpf { )) } + /// Poll the stack-record ring buffer into `tx`. + pub(crate) fn poll_stacks( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + use crate::ebpf::stacks::events; + use runner_shared::artifacts::MemtrackEventKind; + + // The poller outlives this borrow of the skeleton, so the chain lookup + // needs an owned handle rather than a reference to the skeleton map. + let stack_traces = with_skel!(self, skel => { + libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) + .context("Failed to create handle for stack_traces map")? + }); + + let parse = move |data: &[u8]| { + let (mut event, stackid) = events::parse_stack(data)?; + if let MemtrackEventKind::Stack { record } = &mut event.kind { + record.fp_chain = events::fp_chain(&stack_traces, stackid); + } + Some(event) + }; + + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.stacks, + parse, + tx, + poll_interval_ms, + )) + } + /// Poll the exec-mapping request ring buffer into `tx`. Same contract as /// [`Self::poll_events_with_channel`]. pub(crate) fn poll_attach_with_channel( diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 4841049c8..0329cb8d6 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,5 +1,7 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; +use crate::ebpf::stacks::config::stack_copy_size_from_env; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; @@ -7,6 +9,7 @@ use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; @@ -23,6 +26,10 @@ pub struct TrackerOptions { /// Uprobe attach mechanism. `None` detects it from BPF token availability. #[builder(default, setter(strip_option))] pub variant: Option, + /// Bytes of user stack to copy for each allocation event. `None` leaves + /// stack capture off; values are clamped to the supported range. + #[builder(default = None)] + pub stack_copy_size: Option, } impl TrackerOptions { @@ -33,6 +40,7 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .physical(std::env::var("CODSPEED_MEMTRACK_TRACK_PHYSICAL").is_ok_and(|v| v == "1")) + .stack_copy_size(stack_copy_size_from_env()) .build() } } @@ -41,6 +49,9 @@ pub struct Tracker { bpf: Arc>, worker: Mutex>, allocators: bool, + /// The dedup gate spans the whole BPF object, so a second session would + /// reference stack records the first one already consumed. + stacks_polled: Option, } impl Tracker { @@ -51,6 +62,7 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { + Self::bump_memlock_rlimit()?; let mut bpf = MemtrackBpf::load(options)?; @@ -70,6 +82,10 @@ impl Tracker { bpf, worker: Mutex::new(worker), allocators: options.allocators, + stacks_polled: options + .stack_copy_size + .is_some() + .then(|| AtomicBool::new(false)), }) } @@ -82,6 +98,14 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { + let capture_stacks = match &self.stacks_polled { + Some(polled) if polled.swap(true, Ordering::Relaxed) => { + bail!("stack capture supports a single spawned command per tracker") + } + Some(_) => true, + None => false, + }; + let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); @@ -89,6 +113,7 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; + match self.worker.lock().as_ref() { Some(worker) => worker.set_root_pid(pid), // No watcher to arm means exec mappings would be missed. @@ -97,14 +122,17 @@ impl Tracker { } let (tx, rx) = mpsc::channel(); - let poller = { + let (poller, stack_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; - bpf.poll_events_with_channel(10, tx)? + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + (bpf.poll_events_with_channel(10, tx)?, stack_poller) }; resume(pid)?; - Ok(Session::new(child, rx, poller)) + Ok(Session::new(child, rx, poller, stack_poller)) } /// Enable allocator-event tracking in the BPF program. Lifetime events @@ -125,6 +153,11 @@ impl Tracker { self.bpf.lock().dropped_events_count() } + /// Per-cause counts of stack captures that were skipped or truncated. + pub fn stack_capture_stats(&self) -> Result { + self.bpf.lock().stack_capture_stats() + } + /// Only meaningful while the BPF object is alive; teardown frees the maps. pub fn ownership_maps(&self) -> Result { self.bpf.lock().ownership_maps() diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 283cff194..d9a6411b1 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -159,6 +159,13 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; + if tracker.stack_capture_enabled() { + let stats = tracker + .stack_capture_stats() + .context("Failed to read stack capture stats")?; + debug!("stack capture stats: {stats:?}"); + } + // Detach probes explicitly: the IPC thread still holds an Arc clone, so the // tracker would otherwise never be dropped before process::exit and the // kernel would close every link fd serially during exit. diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 7aec33feb..9bed66b7d 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -10,6 +10,7 @@ pub struct Session { child: Child, events: Option>, _poller: RingBufferPoller, + _stack_poller: Option, } impl Session { @@ -17,11 +18,13 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, + stack_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, + _stack_poller: stack_poller, } } From a78b2a2041fd998c4d06c695cc5c57bb78509e0c Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:01:18 +0200 Subject: [PATCH 04/21] test(memtrack): cover allocation stack capture Add a fixture with two non-inlinable malloc call paths and privileged tests over it: distinct call paths get distinct identities with module mappings for the binary and libc, repeated calls deduplicate, and the default-off path still reports allocations. Two cases guard failure modes the default budget cannot reach. The maximum copy budget is the only configuration that exercises the verifier's instruction limit, since the frozen rodata makes the copy and hash loops scale with the configured size. Shrinking the frame-pointer map to one slot proves exhaustion costs only the fallback chain, never an allocation event. Refs COD-3222 --- .github/workflows/ci.yml | 2 +- crates/memtrack/testdata/stack_paths.c | 46 ++++ crates/memtrack/tests/c_tests.rs | 2 +- crates/memtrack/tests/rss_tests.rs | 22 +- crates/memtrack/tests/shared.rs | 21 +- .../stack_tests__nested_doubling.snap | 12 + ...ck_tests__nested_doubling_shared_free.snap | 12 + .../stack_tests__stack_capture_disabled.snap | 206 +++++++++++++++ .../snapshots/stack_tests__stack_paths.snap | 206 +++++++++++++++ crates/memtrack/tests/stack_tests.rs | 240 ++++++++++++++++++ 10 files changed, 759 insertions(+), 10 deletions(-) create mode 100644 crates/memtrack/testdata/stack_paths.c create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap create mode 100644 crates/memtrack/tests/stack_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fe75eac..3265a9194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests, stack_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/crates/memtrack/testdata/stack_paths.c b/crates/memtrack/testdata/stack_paths.c new file mode 100644 index 000000000..0cee7f437 --- /dev/null +++ b/crates/memtrack/testdata/stack_paths.c @@ -0,0 +1,46 @@ +#include +#include + +static volatile void *escaped_pointer; +static volatile unsigned int remaining_a = 50; +static volatile unsigned int remaining_b = 50; + +__attribute__((noinline)) static void path_a_inner(void) { + void *pointer = malloc(64); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_a(void) { + while (remaining_a != 0) { + path_a_inner(); + --remaining_a; + } +} + +__attribute__((noinline)) static void path_b_inner(void) { + void *pointer = malloc(192); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_b(void) { + while (remaining_b != 0) { + path_b_inner(); + --remaining_b; + } +} + +int main(void) { + void *marker_before = malloc(0xC0D59EED); + escaped_pointer = marker_before; + free(marker_before); + + path_a(); + path_b(); + + void *marker_after = malloc(0xC0D59EED); + escaped_pointer = marker_after; + free(marker_after); + return 0; +} diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index 2d33dccf0..d217ff392 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -89,7 +89,7 @@ fn test_track_allocators_disabled_skips_allocations() -> Result<(), Box TrackerOptions { + TrackerOptions::builder() + .allocators(false) + .physical(true) + .build() +} + /// Run a fixture under `track` and return the raw `/proc` RSS report it wrote to /// its argv[1] alongside the collected events. /// @@ -349,7 +357,9 @@ fn test_rss_rmap_tracking( #[case] source: &str, #[case] name: &str, ) -> Result<(), Box> { - let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let (raw_report, events) = track_fixture(source, name, |command| { + shared::track_command_with_opts(command, rmap_only_options()) + })?; let raw_report = raw_report.ok_or("fixture wrote no rss report")?; let (rss_stat, rmap) = per_pid_peaks(&events); let summary = RssSummary { @@ -425,10 +435,14 @@ enum Reclaim { #[case::rss_stat(Reclaim::RssStat)] #[case::rmap(Reclaim::Rmap)] fn test_rss_external_reclaim(#[case] mode: Reclaim) -> Result<(), Box> { + let options = match mode { + Reclaim::RssStat => TrackerOptions::default(), + Reclaim::Rmap => rmap_only_options(), + }; let (_report, events) = track_fixture( include_str!("../testdata/rss/madvise_extern.c"), "madvise_extern", - shared::track_command_with_rmap, + |command| shared::track_command_with_opts(command, options), )?; // A = owner that faulted the file region; B = external caller, single-threaded @@ -597,7 +611,7 @@ fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box Result<(), Box String { match kind { + MemtrackEventKind::Free { .. } => "Free".to_string(), + MemtrackEventKind::Malloc { size, .. } => format!("Malloc {{ size: {size} }}"), + MemtrackEventKind::Calloc { size, .. } => format!("Calloc {{ size: {size} }}"), + MemtrackEventKind::AlignedAlloc { size, .. } => format!("AlignedAlloc {{ size: {size} }}"), MemtrackEventKind::Realloc { size, .. } => format!("Realloc {{ size: {size} }}"), other => format!("{other:?}"), } @@ -124,7 +127,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit - + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -238,6 +241,16 @@ pub fn track_command_with_rmap_maps( Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } +/// Track a command with allocation stack capture enabled, returning its events. +pub fn track_command_with_stacks(command: Command, copy_size: u32) -> TrackResult { + track_command_with_opts( + command, + TrackerOptions::builder() + .stack_copy_size(Some(copy_size)) + .build(), + ) +} + /// Track a command with rmap hooks and snapshot the ownership maps at a /// fixture-signalled checkpoint. /// diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap new file mode 100644 index 000000000..f1d65a997 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap new file mode 100644 index 000000000..f1d65a997 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap new file mode 100644 index 000000000..cf9fc935d --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap new file mode 100644 index 000000000..7cb31d31e --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs new file mode 100644 index 000000000..7a95cd6ca --- /dev/null +++ b/crates/memtrack/tests/stack_tests.rs @@ -0,0 +1,240 @@ +#[macro_use] +mod shared; + +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::collections::HashSet; +use std::process::Command; +use tempfile::TempDir; + +const COPY_SIZE: u32 = memtrack::DEFAULT_STACK_COPY_SIZE; + +fn compile_fixture( + name: &str, + temp_dir: &TempDir, +) -> Result> { + shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + name, + temp_dir.path(), + ) +} +fn require_mapping_support() -> bool { + if memtrack::MappingSupport::detect() == memtrack::MappingSupport::Unsupported { + eprintln!("skipping stack capture test: mapping support is unavailable"); + return false; + } + true +} + +/// The stack identity carried by each allocation and deallocation event that has one. +fn event_hashes(events: &[MemtrackEvent]) -> Vec { + events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } + | MemtrackEventKind::Free { stack_hash } => (stack_hash != 0).then_some(stack_hash), + _ => None, + }) + .collect() +} + +fn record_hashes(events: &[MemtrackEvent]) -> HashSet { + events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record } => Some(record.hash), + _ => None, + }) + .collect() +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + + let records: Vec<_> = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record: r } => { + Some((r.hash, r.sp, &r.regs, &r.bytes, r.truncated)) + } + _ => None, + }) + .collect(); + + assert!( + records.len() >= 2, + "expected at least two stack records, got {} ({} events)", + records.len(), + events.len() + ); + + let hashes = record_hashes(&events); + assert_eq!( + hashes.len(), + records.len(), + "stack records must be deduplicated by unique hash" + ); + + for (hash, sp, regs, bytes, truncated) in &records { + assert_ne!(*sp, 0, "record {hash:#x} has no stack pointer"); + assert_eq!(regs.len(), 33, "record {hash:#x} must carry 33 registers"); + assert!( + !bytes.is_empty() && bytes.len() % 512 == 0 && bytes.len() <= COPY_SIZE as usize, + "record {hash:#x} must hold whole 512-byte chunks within the budget, got {}", + bytes.len() + ); + assert_eq!( + *truncated, + bytes.len() == COPY_SIZE as usize, + "record {hash:#x} may only be flagged truncated when it filled the budget" + ); + } + + let carried = event_hashes(&events); + assert!( + !carried.is_empty(), + "expected events carrying a captured stack hash" + ); + assert!( + carried.iter().all(|hash| hashes.contains(hash)), + "every non-zero stack_hash must have a matching stack record" + ); + + // The fixture frees every allocation, so both sides must report identities. + assert!( + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Free { stack_hash } if stack_hash != 0)), + "free events must carry their own stack identity" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn dedup_collapses_repeated_call_paths() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + + let carried = event_hashes(&events); + let records = record_hashes(&events); + assert!( + carried.len() > records.len(), + "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", + carried.len(), + records.len(), + events.len() + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// The largest budget stresses the verifier hardest: the copy loop and its +/// unrolled per-chunk hash both scale with the configured size, so a program +/// that loads at the default can still exceed the instruction limit here. +/// It is also the only budget at which nothing can be budget-limited, because +/// the stack mapping always ends first. +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn max_copy_budget_loads_and_captures_whole_stacks() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_max", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), u32::MAX)?; + + let truncated: Vec<_> = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record: r } if r.truncated => Some(r.hash), + _ => None, + }) + .collect(); + + assert!( + !record_hashes(&events).is_empty(), + "expected stack records at the maximum copy budget" + ); + assert!( + truncated.is_empty(), + "no capture can be budget-limited at the maximum budget: {truncated:#x?}" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// Restores the capture toggle on drop so a failing assertion cannot leak the +/// override into later tests (the suite runs single-threaded). +struct DisableCaptureGuard; + +impl DisableCaptureGuard { + fn set() -> Self { + // SAFETY: tests run with --test-threads 1, so no concurrent env access. + unsafe { std::env::set_var("CODSPEED_MEMTRACK_CAPTURE_STACKS", "0") }; + Self + } +} + +impl Drop for DisableCaptureGuard { + fn drop(&mut self) { + // SAFETY: see `set`. + unsafe { std::env::remove_var("CODSPEED_MEMTRACK_CAPTURE_STACKS") }; + } +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_disabled", &temp_dir)?; + let _guard = DisableCaptureGuard::set(); + let (events, thread_handle) = shared::track_binary(&binary)?; + + assert!( + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Malloc { .. })), + "disabled capture must still report allocation events" + ); + assert!( + record_hashes(&events).is_empty(), + "disabled capture must emit zero stack records" + ); + assert!( + event_hashes(&events).is_empty(), + "disabled capture must leave stack_hash zero on every event" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} From 18eea27251e4a64529f0c581ae0437725984d93c Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 19:44:42 +0200 Subject: [PATCH 05/21] refactor(runner): move ELF artifact pipeline to executor/shared The symbol, unwind-data and debug-info extraction is not perf-specific: it turns a set of mapped ELF modules into the deduplicated keyed artifacts a profile references, whatever discovered the mappings. Memory mode needs the same pipeline, so it moves out of wall_time/profiler/perf into executor/shared/module_artifacts. --- src/executor/shared/mod.rs | 1 + .../perf => shared/module_artifacts}/debug_info.rs | 0 .../perf => shared/module_artifacts}/elf_helper.rs | 0 .../module_artifacts}/loaded_module.rs | 0 src/executor/shared/module_artifacts/mod.rs | 13 +++++++++++++ .../module_artifacts}/module_symbols.rs | 0 .../perf => shared/module_artifacts}/naming.rs | 0 .../module_artifacts}/save_artifacts.rs | 2 +- ...tifacts__debug_info__tests__cpp_debug_info.snap} | 2 +- ...acts__debug_info__tests__golang_debug_info.snap} | 2 +- ...ifacts__debug_info__tests__ruff_debug_info.snap} | 2 +- ...__debug_info__tests__rust_divan_debug_info.snap} | 2 +- ...bug_info__tests__the_algorithms_debug_info.snap} | 2 +- ...ifacts__module_symbols__tests__cpp_symbols.snap} | 2 +- ...cts__module_symbols__tests__golang_symbols.snap} | 2 +- ...facts__module_symbols__tests__ruff_symbols.snap} | 2 +- ..._module_symbols__tests__rust_divan_symbols.snap} | 2 +- ...ule_symbols__tests__the_algorithms_symbols.snap} | 2 +- ...facts__unwind_data__tests__cpp_unwind_data.snap} | 2 +- ...ts__unwind_data__tests__golang_unwind_data.snap} | 2 +- ...acts__unwind_data__tests__ruff_unwind_data.snap} | 2 +- ...unwind_data__tests__rust_divan_unwind_data.snap} | 2 +- ...nd_data__tests__the_algorithms_unwind_data.snap} | 2 +- .../perf => shared/module_artifacts}/unwind_data.rs | 0 src/executor/wall_time/profiler/perf/jit_dump.rs | 2 +- src/executor/wall_time/profiler/perf/mod.rs | 8 +------- .../wall_time/profiler/perf/parse_perf_file.rs | 6 +++--- 27 files changed, 35 insertions(+), 27 deletions(-) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/debug_info.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/elf_helper.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/loaded_module.rs (100%) create mode 100644 src/executor/shared/module_artifacts/mod.rs rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/module_symbols.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/naming.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/save_artifacts.rs (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/unwind_data.rs (100%) diff --git a/src/executor/shared/mod.rs b/src/executor/shared/mod.rs index 2badf4064..f278f07cd 100644 --- a/src/executor/shared/mod.rs +++ b/src/executor/shared/mod.rs @@ -1 +1,2 @@ pub mod fifo; +pub mod module_artifacts; diff --git a/src/executor/wall_time/profiler/perf/debug_info.rs b/src/executor/shared/module_artifacts/debug_info.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/debug_info.rs rename to src/executor/shared/module_artifacts/debug_info.rs diff --git a/src/executor/wall_time/profiler/perf/elf_helper.rs b/src/executor/shared/module_artifacts/elf_helper.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/elf_helper.rs rename to src/executor/shared/module_artifacts/elf_helper.rs diff --git a/src/executor/wall_time/profiler/perf/loaded_module.rs b/src/executor/shared/module_artifacts/loaded_module.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/loaded_module.rs rename to src/executor/shared/module_artifacts/loaded_module.rs diff --git a/src/executor/shared/module_artifacts/mod.rs b/src/executor/shared/module_artifacts/mod.rs new file mode 100644 index 000000000..34528e0ad --- /dev/null +++ b/src/executor/shared/module_artifacts/mod.rs @@ -0,0 +1,13 @@ +//! Extract symbols, unwind data, and debug info from mapped ELF modules. +//! +//! The input is a set of [`loaded_module::LoadedModule`] values. The output is +//! keyed `unwind_data`/`symbols.map` files and per-process metadata references. + +mod elf_helper; +mod naming; + +pub mod debug_info; +pub mod loaded_module; +pub mod module_symbols; +pub mod save_artifacts; +pub mod unwind_data; diff --git a/src/executor/wall_time/profiler/perf/module_symbols.rs b/src/executor/shared/module_artifacts/module_symbols.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/module_symbols.rs rename to src/executor/shared/module_artifacts/module_symbols.rs diff --git a/src/executor/wall_time/profiler/perf/naming.rs b/src/executor/shared/module_artifacts/naming.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/naming.rs rename to src/executor/shared/module_artifacts/naming.rs diff --git a/src/executor/wall_time/profiler/perf/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs similarity index 99% rename from src/executor/wall_time/profiler/perf/save_artifacts.rs rename to src/executor/shared/module_artifacts/save_artifacts.rs index 36b2fd12a..9b4899427 100644 --- a/src/executor/wall_time/profiler/perf/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -1,7 +1,7 @@ use super::debug_info::debug_info_by_path; use super::loaded_module::LoadedModule; +use super::naming; use crate::executor::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore; -use crate::executor::wall_time::profiler::perf::naming; use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap index 48d654070..9b917e545 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap index e92dcefa8..5b6a04ae2 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap index 75d0a4494..97dc7b43a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap index 6cf90c6a1..fd5e1aa0c 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap index 9e9c52a2e..a238cbb28 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap index 8456dd05b..34c79e04e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap index 84138e7bb..990e660d9 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap index 879d29f90..fe5907dd0 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap index 839039f35..10a77b716 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap index fec3e2802..724f3002e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap index 205b5e148..6c554024a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap index 699e4b031..807e10611 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap index a0a5b0f98..3956fd7d1 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap index 0367c2dee..edfd8e558 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap index 9fc15dca2..066c1ad0e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/unwind_data.rs b/src/executor/shared/module_artifacts/unwind_data.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/unwind_data.rs rename to src/executor/shared/module_artifacts/unwind_data.rs diff --git a/src/executor/wall_time/profiler/perf/jit_dump.rs b/src/executor/wall_time/profiler/perf/jit_dump.rs index fd5fad056..344f4080e 100644 --- a/src/executor/wall_time/profiler/perf/jit_dump.rs +++ b/src/executor/wall_time/profiler/perf/jit_dump.rs @@ -1,4 +1,4 @@ -use super::module_symbols::{ModuleSymbols, Symbol}; +use crate::executor::shared::module_artifacts::module_symbols::{ModuleSymbols, Symbol}; use crate::prelude::*; use linux_perf_data::jitdump::{JitDumpReader, JitDumpRecord}; use runner_shared::unwind_data::{ProcessUnwindData, UnwindData}; diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 8816e5fb0..625407e86 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -10,6 +10,7 @@ use crate::executor::helpers::env::suppress_go_perf_unwinding_warning; use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids; use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; +use crate::executor::shared::module_artifacts::save_artifacts; use crate::executor::wall_time::profiler::NO_BENCHMARKS_DETECTED_WARNING; use crate::executor::wall_time::profiler::Profiler; use crate::executor::wall_time::profiler::SAMPLING_RATE_HZ; @@ -29,16 +30,9 @@ use runner_shared::metadata::WalltimeMetadata; use std::path::Path; use std::path::PathBuf; -mod debug_info; -mod elf_helper; mod jit_dump; -mod loaded_module; -mod module_symbols; -mod naming; mod parse_perf_file; -mod save_artifacts; pub(crate) mod setup; -mod unwind_data; pub mod fifo; pub mod perf_executable; diff --git a/src/executor/wall_time/profiler/perf/parse_perf_file.rs b/src/executor/wall_time/profiler/perf/parse_perf_file.rs index 151b54945..1d1033b38 100644 --- a/src/executor/wall_time/profiler/perf/parse_perf_file.rs +++ b/src/executor/wall_time/profiler/perf/parse_perf_file.rs @@ -1,6 +1,6 @@ -use super::loaded_module::{LoadedModule, ProcessLoadedModule}; -use super::module_symbols::ModuleSymbols; -use super::unwind_data::unwind_data_from_elf; +use crate::executor::shared::module_artifacts::loaded_module::{LoadedModule, ProcessLoadedModule}; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; use libc::pid_t; use linux_perf_data::PerfFileReader; From 09e91ae6735339b640c8727cb9c57f99f217a79a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 19:53:32 +0200 Subject: [PATCH 06/21] feat(runner): add MemtrackMetadata sharing ModuleArtifacts with walltime Memory mode needs the same per-pid module references walltime writes, so the five artifact fields move into a flattened `ModuleArtifacts` shared by both formats; walltime's JSON is unchanged, asserted against output captured from the flat struct. Flattening buffers those fields through serde's `Content`, which unlike serde_json's direct deserializer cannot parse a string JSON key into a pid, so pid-keyed maps get an explicit key-parsing helper. --- crates/runner-shared/src/lib.rs | 1 + crates/runner-shared/src/metadata.rs | 191 ++++++++++++++++-- crates/runner-shared/src/serde_pid_map.rs | 36 ++++ .../shared/module_artifacts/save_artifacts.rs | 25 +-- src/executor/wall_time/profiler/perf/mod.rs | 6 +- src/executor/wall_time/profiler/samply/mod.rs | 6 +- 6 files changed, 229 insertions(+), 36 deletions(-) create mode 100644 crates/runner-shared/src/serde_pid_map.rs diff --git a/crates/runner-shared/src/lib.rs b/crates/runner-shared/src/lib.rs index 61e804de7..2cdc7d5a6 100644 --- a/crates/runner-shared/src/lib.rs +++ b/crates/runner-shared/src/lib.rs @@ -4,5 +4,6 @@ pub mod fifo; pub mod metadata; pub mod module_symbols; pub mod perf_event; +pub mod serde_pid_map; pub mod unwind_data; pub mod walltime_results; diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 7a2c7c89b..654ae298d 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -11,35 +11,43 @@ use crate::fifo::MarkerType; use crate::module_symbols::MappedProcessModuleSymbols; use crate::unwind_data::MappedProcessUnwindData; +/// The per-profile module artifacts: the deduplicated debug info, unwind data +/// and symbol tables extracted from the ELF modules the profiled processes +/// mapped, plus the per-pid references into them. +/// +/// Flattened into every metadata format, so all profiling modes describe their +/// modules identically. #[derive(Serialize, Deserialize, Default)] -pub struct WalltimeMetadata { - /// The version of this metadata format. - pub version: u64, - - /// Name and version of the integration - pub integration: (String, String), - - /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub ignored_modules_by_pid: HashMap>, - +pub struct ModuleArtifacts { /// Deduplicated debug info entries, keyed by semantic key #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub debug_info: HashMap, /// Per-pid debug info references, mapping PID to mounted modules' debug info /// Referenced by `path_keys` that point to the deduplicated `debug_info` entries. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_debug_info_by_pid: HashMap>, /// Per-pid unwind data references, mapping PID to mounted modules' unwind data /// Referenced by `path_keys` that point to the deduplicated `unwind_data` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_unwind_data_by_pid: HashMap>, /// Per-pid symbol references, mapping PID to its mounted modules' symbols /// Referenced by `path_keys` that point to the deduplicated `symbols.map` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_module_symbols: HashMap>, /// Mapping from semantic `path_key` to original binary path on host disk @@ -49,6 +57,22 @@ pub struct WalltimeMetadata { /// Until now, only kept for traceability, if we ever need to reconstruct the original paths from the keys #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub path_key_to_path: HashMap, +} + +#[derive(Serialize, Deserialize, Default)] +pub struct WalltimeMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub ignored_modules_by_pid: HashMap>, + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, // Deprecated fields below are kept for backward compatibility, since this struct is used in // the parser and older versions of the runner still generate them @@ -85,3 +109,142 @@ impl WalltimeMetadata { Ok(()) } } + +/// Companion to the memtrack event stream: the modules its allocation stacks +/// resolve against. Memory mode records benchmark boundaries in +/// `ExecutionTimestamps`, so unlike [`WalltimeMetadata`] it carries no markers. +#[derive(Serialize, Deserialize, Default)] +pub struct MemtrackMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, +} + +impl MemtrackMetadata { + pub fn from_reader(reader: R) -> anyhow::Result { + serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") + } + + pub fn save_to>(&self, path: P) -> anyhow::Result<()> { + let file = std::fs::File::create(path.as_ref().join("memtrack.metadata"))?; + const BUFFER_SIZE: usize = 256 * 1024 /* 256 KB */; + + let writer = BufWriter::with_capacity(BUFFER_SIZE, file); + serde_json::to_writer(writer, self)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Captured from the flat `WalltimeMetadata` that predates + /// [`ModuleArtifacts`]: flattening must not move a single byte, since the + /// parser reads this format from runners of every version. + const WALLTIME_JSON: &str = r#"{"version":7,"integration":["codspeed-rust","4.2.0"],"ignored_modules_by_pid":{"42":[["/lib/libpython.so",4096,8192]]},"debug_info":{"0__libc.so.6":{"object_path":"/lib/libc.so.6","addr_bounds":[4096,36864],"load_bias":4096,"debug_infos":[{"addr":4352,"size":32,"name":"malloc","file":"malloc.c","line":11}]}},"mapped_process_debug_info_by_pid":{"42":[{"debug_info_key":"0__libc.so.6","load_bias":4096}]},"mapped_process_unwind_data_by_pid":{"42":[{"unwind_data_key":"0__libc.so.6","timestamp":1234,"avma_range":{"start":4096,"end":36864},"base_avma":4096}]},"mapped_process_module_symbols":{"42":[{"perf_map_key":"0__libc.so.6","load_bias":4096}]},"path_key_to_path":{"0__libc.so.6":"/lib/libc.so.6"},"uri_by_ts":[[1,"bench::a"]],"ignored_modules":[],"markers":[]}"#; + + fn populated_artifacts() -> ModuleArtifacts { + ModuleArtifacts { + debug_info: HashMap::from([( + "0__libc.so.6".to_string(), + ModuleDebugInfo { + object_path: "/lib/libc.so.6".to_string(), + addr_bounds: (0x1000, 0x9000), + load_bias: 0x1000, + debug_infos: vec![crate::debug_info::DebugInfo { + addr: 0x1100, + size: 0x20, + name: "malloc".to_string(), + file: "malloc.c".to_string(), + line: Some(11), + }], + }, + )]), + mapped_process_debug_info_by_pid: HashMap::from([( + 42, + vec![MappedProcessDebugInfo { + debug_info_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + mapped_process_unwind_data_by_pid: HashMap::from([( + 42, + vec![MappedProcessUnwindData { + unwind_data_key: "0__libc.so.6".to_string(), + inner: crate::unwind_data::ProcessUnwindData { + timestamp: Some(1234), + avma_range: 0x1000..0x9000, + base_avma: 0x1000, + }, + }], + )]), + mapped_process_module_symbols: HashMap::from([( + 42, + vec![crate::module_symbols::MappedProcessModuleSymbols { + perf_map_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + path_key_to_path: HashMap::from([( + "0__libc.so.6".to_string(), + PathBuf::from("/lib/libc.so.6"), + )]), + } + } + + #[test] + fn walltime_metadata_serialization_is_unchanged_by_flattening() { + #[allow(deprecated)] + let metadata = WalltimeMetadata { + version: 7, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + ignored_modules_by_pid: HashMap::from([( + 42, + vec![("/lib/libpython.so".to_string(), 0x1000, 0x2000)], + )]), + artifacts: populated_artifacts(), + uri_by_ts: vec![(1, "bench::a".to_string())], + ignored_modules: vec![], + markers: vec![], + debug_info_by_pid: HashMap::new(), + }; + + assert_eq!(serde_json::to_string(&metadata).unwrap(), WALLTIME_JSON); + } + + #[test] + fn walltime_metadata_round_trips_through_the_flattened_fields() { + let parsed = WalltimeMetadata::from_reader(WALLTIME_JSON.as_bytes()).unwrap(); + + assert_eq!(parsed.artifacts.path_key_to_path.len(), 1); + assert_eq!( + parsed.artifacts.mapped_process_unwind_data_by_pid[&42].len(), + 1 + ); + assert_eq!(serde_json::to_string(&parsed).unwrap(), WALLTIME_JSON); + } + + #[test] + fn memtrack_metadata_round_trips() { + let metadata = MemtrackMetadata { + version: 1, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + artifacts: populated_artifacts(), + }; + + let json = serde_json::to_string(&metadata).unwrap(); + let parsed = MemtrackMetadata::from_reader(json.as_bytes()).unwrap(); + + assert_eq!(serde_json::to_string(&parsed).unwrap(), json); + assert_eq!( + parsed.artifacts.mapped_process_module_symbols[&42][0].perf_map_key, + "0__libc.so.6" + ); + } +} diff --git a/crates/runner-shared/src/serde_pid_map.rs b/crates/runner-shared/src/serde_pid_map.rs new file mode 100644 index 000000000..fa6fbb9dc --- /dev/null +++ b/crates/runner-shared/src/serde_pid_map.rs @@ -0,0 +1,36 @@ +//! `#[serde(with = ...)]` support for pid-keyed maps. +//! +//! JSON object keys are always strings. serde_json's direct deserializer +//! special-cases that and parses integer map keys, but a `#[serde(flatten)]` +//! field is buffered into serde's internal `Content` first, and that path has no +//! such special case — an `i32` key then fails with `invalid type: string`. So +//! the keys are read as strings and parsed here, which works on both paths. + +use libc::pid_t; +use serde::de::{Deserializer, Error}; +use serde::{Deserialize, Serialize, Serializer}; +use std::collections::HashMap; + +pub fn serialize(map: &HashMap, serializer: S) -> Result +where + V: Serialize, + S: Serializer, +{ + map.serialize(serializer) +} + +pub fn deserialize<'de, V, D>(deserializer: D) -> Result, D::Error> +where + V: Deserialize<'de>, + D: Deserializer<'de>, +{ + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| { + let pid = key + .parse::() + .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; + Ok((pid, value)) + }) + .collect() +} diff --git a/src/executor/shared/module_artifacts/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs index 9b4899427..3e8903ed9 100644 --- a/src/executor/shared/module_artifacts/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -6,18 +6,17 @@ use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; use runner_shared::debug_info::{MappedProcessDebugInfo, ModuleDebugInfo}; +use runner_shared::metadata::ModuleArtifacts; use runner_shared::module_symbols::MappedProcessModuleSymbols; use runner_shared::unwind_data::{MappedProcessUnwindData, ProcessUnwindData, UnwindData}; use std::collections::HashMap; use std::path::{Path, PathBuf}; pub struct SavedArtifacts { - pub symbol_pid_mappings_by_pid: HashMap>, - pub debug_info: HashMap, - pub mapped_process_debug_info_by_pid: HashMap>, - pub mapped_process_unwind_data_by_pid: HashMap>, + pub artifacts: ModuleArtifacts, + /// Kept out of [`ModuleArtifacts`] because only the folded walltime trace + /// drops modules; other modes carry every module they mapped. pub ignored_modules_by_pid: HashMap>, - pub key_to_path: HashMap, } /// Save all artifacts (symbols, debug info, unwind data) from mounted modules and JIT data. @@ -30,7 +29,7 @@ pub fn save_artifacts( register_paths(&mut path_to_key, loaded_modules_by_path); - let symbol_pid_mappings_by_pid = + let mapped_process_module_symbols = save_symbols(profile_folder, loaded_modules_by_path, &path_to_key); let (debug_info, mapped_process_debug_info_by_pid) = @@ -45,18 +44,20 @@ pub fn save_artifacts( let ignored_modules_by_pid = collect_ignored_modules(loaded_modules_by_path); - let key_to_path = path_to_key + let path_key_to_path = path_to_key .into_iter() .map(|(path, key)| (key, path)) .collect(); SavedArtifacts { - symbol_pid_mappings_by_pid, - debug_info, - mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid, + artifacts: ModuleArtifacts { + debug_info, + mapped_process_debug_info_by_pid, + mapped_process_unwind_data_by_pid, + mapped_process_module_symbols, + path_key_to_path, + }, ignored_modules_by_pid, - key_to_path, } } diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 625407e86..7f31921e2 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -300,11 +300,7 @@ impl BenchmarkData<'_> { uri_by_ts: self.marker_result.uri_by_ts.clone(), ignored_modules_by_pid: artifacts.ignored_modules_by_pid, markers: self.marker_result.markers.clone(), - debug_info: artifacts.debug_info, - mapped_process_debug_info_by_pid: artifacts.mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid: artifacts.mapped_process_unwind_data_by_pid, - mapped_process_module_symbols: artifacts.symbol_pid_mappings_by_pid, - path_key_to_path: artifacts.key_to_path, + artifacts: artifacts.artifacts, // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), ignored_modules: Default::default(), diff --git a/src/executor/wall_time/profiler/samply/mod.rs b/src/executor/wall_time/profiler/samply/mod.rs index 3d77e7ade..5f04ef8c8 100644 --- a/src/executor/wall_time/profiler/samply/mod.rs +++ b/src/executor/wall_time/profiler/samply/mod.rs @@ -184,11 +184,7 @@ impl Profiler for SamplyProfiler { // These fields aren't required in samply, since we symbolicate client-side. ignored_modules_by_pid: Default::default(), - debug_info: Default::default(), - mapped_process_debug_info_by_pid: Default::default(), - mapped_process_unwind_data_by_pid: Default::default(), - mapped_process_module_symbols: Default::default(), - path_key_to_path: Default::default(), + artifacts: Default::default(), // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), From 96b7bbe31cc9599fab7c0d66575d28a1aa1360a7 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:13:30 +0200 Subject: [PATCH 07/21] feat(memtrack): record mapped modules for offline stack attribution Allocation stacks are raw addresses, so resolving them off-box needs the module geometry perf gets from PERF_RECORD_MMAP2. No single hook provides it: security_mmap_file has the file but runs before the VMA exists, and perf_event_mmap has the addresses but cannot resolve a path. So an LSM program caches the path once per inode and a perf_event_mmap fentry emits inode-keyed address records, joined in userspace while the maps are still live. Path resolution is only reachable from an LSM program at all, and only above 5.11 (bpf_d_path on the sleepable hook) or 6.12 (the bpf_path_d_path kfunc), with the bpf LSM active. MappingSupport probes both, and when neither holds stack capture is turned off rather than shipping stacks nothing can attribute. --- crates/memtrack/src/ebpf/c/attach.h | 5 - crates/memtrack/src/ebpf/c/event.h | 21 +++ crates/memtrack/src/ebpf/c/main.bpf.c | 1 + crates/memtrack/src/ebpf/c/mappings.bpf.h | 161 ++++++++++++++++++ crates/memtrack/src/ebpf/mappings/mod.rs | 7 + crates/memtrack/src/ebpf/mappings/records.rs | 83 +++++++++ crates/memtrack/src/ebpf/mappings/resolve.rs | 73 ++++++++ crates/memtrack/src/ebpf/mappings/support.rs | 103 +++++++++++ crates/memtrack/src/ebpf/memtrack/maps.rs | 52 ++++++ crates/memtrack/src/ebpf/memtrack/mod.rs | 57 ++++++- crates/memtrack/src/ebpf/memtrack/tracking.rs | 23 +++ crates/memtrack/src/ebpf/mod.rs | 3 +- crates/memtrack/src/ebpf/tracker.rs | 83 +++++++-- crates/memtrack/src/session.rs | 3 + crates/memtrack/tests/shared.rs | 6 +- crates/memtrack/tests/stack_tests.rs | 47 +---- .../src/artifacts/memtrack/mappings.rs | 37 ++++ .../src/artifacts/memtrack/mod.rs | 2 + 18 files changed, 692 insertions(+), 75 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/mappings.bpf.h create mode 100644 crates/memtrack/src/ebpf/mappings/mod.rs create mode 100644 crates/memtrack/src/ebpf/mappings/records.rs create mode 100644 crates/memtrack/src/ebpf/mappings/resolve.rs create mode 100644 crates/memtrack/src/ebpf/mappings/support.rs create mode 100644 crates/runner-shared/src/artifacts/memtrack/mappings.rs diff --git a/crates/memtrack/src/ebpf/c/attach.h b/crates/memtrack/src/ebpf/c/attach.h index e188c7d5f..90cbe4360 100644 --- a/crates/memtrack/src/ebpf/c/attach.h +++ b/crates/memtrack/src/ebpf/c/attach.h @@ -14,11 +14,6 @@ #define MEMTRACK_PROT_EXEC 0x4 #define MEMTRACK_SIGSTOP 19 -struct inode_key { - __u64 dev; - __u64 ino; -}; - /* (dev, ino) -> 1; populated by userspace after classify/attach */ BPF_HASH_MAP(known_inodes, struct inode_key, __u8, 8192); /* Requests are 24 B and rare; overflow aborts the run via the counter below */ diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 7e10a7756..e3fdab0aa 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -115,6 +115,13 @@ struct event { } data; }; +/* Identifies a mapped file across both the attach watcher and the mapping + * recorder. `dev` uses the kernel's s_dev encoding: (major << 20) | minor. */ +struct inode_key { + uint64_t dev; + uint64_t ino; +}; + /* Request from the exec-mapping watcher to the userspace attach worker */ struct attach_request { uint32_t pid; @@ -122,4 +129,18 @@ struct attach_request { uint64_t ino; }; +/* One executable file mapping, mirroring PERF_RECORD_MMAP2. The path is not + * here: it is resolved once per inode into a BPF map that userspace joins + * against, since every mapping of the same file shares it. */ +struct mapping_record { + uint64_t dev; + uint64_t ino; + uint64_t file_offset; /* offset of the mapping's first byte in the file */ + uint64_t start; + uint64_t end; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + uint32_t pid; + uint32_t _pad; +}; + #endif /* __EVENT_H__ */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index b405f572b..7a068c605 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,6 +8,7 @@ #include "allocator.h" #include "attach.h" #include "event.h" +#include "mappings.bpf.h" #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" diff --git a/crates/memtrack/src/ebpf/c/mappings.bpf.h b/crates/memtrack/src/ebpf/c/mappings.bpf.h new file mode 100644 index 000000000..5b5635a12 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/mappings.bpf.h @@ -0,0 +1,161 @@ +#ifndef __MAPPINGS_BPF_H__ +#define __MAPPINGS_BPF_H__ + +#include "event.h" +#include "utils/folio.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* == Mapping recorder == + * + * Reconstructs what `PERF_RECORD_MMAP2` gives perf: which file a tracked + * process mapped, where, so raw stack addresses can be attributed to modules + * offline. No single hook carries both halves: + * + * security_mmap_file(file, ..) has the file, runs before the VMA exists + * perf_event_mmap(vma) has the addresses, cannot resolve a path + * + * The path therefore lands in a per-inode cache, and the address-bearing hook + * emits inode-keyed records that userspace joins against that cache while this + * BPF object is still loaded. + * + * Path resolution is only reachable from an LSM program: `bpf_d_path()` is + * restricted to sleepable LSM hooks, `BPF_TRACE_ITER` and an fentry allowlist + * holding no mmap path, and the newer `bpf_path_d_path()` kfunc rejects + * non-LSM program types. Both variants are compiled; userspace autoloads the + * one the running kernel supports and neither when the bpf LSM is inactive. */ + +/* VM_EXEC from linux/mm.h, which vmlinux.h does not carry (it is a macro, not a + * type). Only executable mappings are recorded: unwind data and symbols are + * looked up by text address. */ +#define MEMTRACK_VM_EXEC 0x00000004 + +/* d_path() fails with -ENAMETOOLONG rather than truncating, so a short buffer + * loses whole modules. PATH_MAX keeps that from happening. */ +#define MEMTRACK_MAX_PATH 4096 + +struct inode_path { + __u32 len; /* bytes written by d_path, including the NUL */ + char path[MEMTRACK_MAX_PATH]; +}; + +/* Every mapping of an inode shares its cached path. */ +BPF_HASH_MAP(path_by_inode, struct inode_key, struct inode_path, 2048); + +/* A dropped record may leave a module unresolved. */ +BPF_RINGBUF(mappings, 256 * 1024); +BPF_ARRAY_MAP(mapping_dropped, __u64, 1); + +/* The path does not fit on the BPF stack; build it in this per-CPU scratch map. */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct inode_path); +} path_scratch SEC(".maps"); + +extern int bpf_path_d_path(const struct path* path, char* buf, __u64 buf__sz) __ksym __weak; + +static __always_inline void bump_mapping_dropped(void) { + __u32 zero = 0; + __u64* drops = bpf_map_lookup_elem(&mapping_dropped, &zero); + if (drops) { + __sync_fetch_and_add(drops, 1); + } +} + +/* Return a scratch slot when this inode has no cached path. */ +static __always_inline struct inode_path* mapping_path_slot(struct file* file, + struct inode_key* key) { + if (!file || !is_tracked(current_tgid())) { + return NULL; + } + + key->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); + key->ino = BPF_CORE_READ(file, f_inode, i_ino); + if (bpf_map_lookup_elem(&path_by_inode, key)) { + return NULL; + } + + __u32 zero = 0; + return bpf_map_lookup_elem(&path_scratch, &zero); +} + +/* Publish a resolved path. A failed resolution is not cached, so the next + * mapping of the same inode retries instead of losing the module for the run. */ +static __always_inline void commit_mapping_path(struct inode_key* key, struct inode_path* entry, + int len) { + if (len <= 0) { + return; + } + entry->len = (__u32)len; + bpf_map_update_elem(&path_by_inode, key, entry, BPF_NOEXIST); +} + +/* Kernels >= 6.12: the kfunc is callable from any LSM program. */ +SEC("lsm/mmap_file") +int BPF_PROG(cache_mmap_path_kfunc, struct file* file, unsigned long reqprot, unsigned long prot, + unsigned long flags) { + struct inode_key key = {}; + struct inode_path* entry = mapping_path_slot(file, &key); + if (entry) { + commit_mapping_path(&key, entry, + bpf_path_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); + } + return 0; +} + +/* Kernels 5.11..6.11: `bpf_d_path()` needs a sleepable LSM hook, which + * `mmap_file` has been since 5.11. */ +SEC("lsm.s/mmap_file") +int BPF_PROG(cache_mmap_path_legacy, struct file* file, unsigned long reqprot, unsigned long prot, + unsigned long flags) { + struct inode_key key = {}; + struct inode_path* entry = mapping_path_slot(file, &key); + if (entry) { + commit_mapping_path(&key, entry, bpf_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); + } + return 0; +} + +/* The same hook perf emits MMAP2 from, so the recorded geometry matches what + * the walltime pipeline already consumes: the file offset is in bytes, not + * pages. */ +SEC("fentry/perf_event_mmap") +int BPF_PROG(record_mmap, struct vm_area_struct* vma) { + if (!vma) { + return 0; + } + + __u32 tgid = current_tgid(); + if (!is_tracked(tgid)) { + return 0; + } + + struct file* file = BPF_CORE_READ(vma, vm_file); + if (!file) { + return 0; + } + if (!(BPF_CORE_READ(vma, vm_flags) & MEMTRACK_VM_EXEC)) { + return 0; + } + + struct mapping_record* rec = bpf_ringbuf_reserve(&mappings, sizeof(*rec), 0); + if (!rec) { + bump_mapping_dropped(); + return 0; + } + + rec->pid = tgid; + rec->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); + rec->ino = BPF_CORE_READ(file, f_inode, i_ino); + rec->file_offset = (__u64)BPF_CORE_READ(vma, vm_pgoff) << page_shift; + rec->start = BPF_CORE_READ(vma, vm_start); + rec->end = BPF_CORE_READ(vma, vm_end); + rec->timestamp = bpf_ktime_get_ns(); + bpf_ringbuf_submit(rec, 0); + + return 0; +} + +#endif /* __MAPPINGS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/mappings/mod.rs b/crates/memtrack/src/ebpf/mappings/mod.rs new file mode 100644 index 000000000..581de916b --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/mod.rs @@ -0,0 +1,7 @@ +mod records; +mod resolve; +mod support; + +pub(crate) use records::MappingRecord; +pub(crate) use resolve::resolve_mappings; +pub use support::MappingSupport; diff --git a/crates/memtrack/src/ebpf/mappings/records.rs b/crates/memtrack/src/ebpf/mappings/records.rs new file mode 100644 index 000000000..4c660031e --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/records.rs @@ -0,0 +1,83 @@ +use crate::ebpf::events::bindings::mapping_record; + +/// One executable file mapping as the BPF recorder saw it. The path is resolved +/// separately, per inode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MappingRecord { + pub pid: u32, + pub dev: u64, + pub ino: u64, + pub file_offset: u64, + pub start: u64, + pub end: u64, + pub timestamp: u64, +} + +impl MappingRecord { + pub fn parse(data: &[u8]) -> Option { + if data.len() < std::mem::size_of::() { + return None; + } + + // SAFETY: the length is checked above, and the layout is the + // bindgen-generated C ABI struct. + let record: mapping_record = unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; + Some(Self { + pid: record.pid, + dev: record.dev, + ino: record.ino, + file_offset: record.file_offset, + start: record.start, + end: record.end, + timestamp: record.timestamp, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encode(record: mapping_record) -> Vec { + // SAFETY: reading a plain-data struct as bytes. + unsafe { + std::slice::from_raw_parts( + (&record as *const mapping_record).cast::(), + std::mem::size_of::(), + ) + } + .to_vec() + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let bytes = encode(mapping_record { + dev: 0x1_0002, + ino: 4242, + file_offset: 0x2000, + start: 0x5555_5555_0000, + end: 0x5555_5556_0000, + timestamp: 987_654_321, + pid: 7, + _pad: 0, + }); + + assert_eq!( + MappingRecord::parse(&bytes), + Some(MappingRecord { + pid: 7, + dev: 0x1_0002, + ino: 4242, + file_offset: 0x2000, + start: 0x5555_5555_0000, + end: 0x5555_5556_0000, + timestamp: 987_654_321, + }) + ); + } + + #[test] + fn truncated_buffer_returns_none() { + assert!(MappingRecord::parse(&[0u8; 8]).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/mappings/resolve.rs b/crates/memtrack/src/ebpf/mappings/resolve.rs new file mode 100644 index 000000000..9c20ba7f1 --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/resolve.rs @@ -0,0 +1,73 @@ +use super::MappingRecord; +use crate::prelude::*; +use runner_shared::artifacts::ProcessMapping; +use std::collections::HashMap; + +/// Records without a path are dropped because their unwind data and symbols cannot be read. +pub(crate) fn resolve_mappings( + records: Vec, + paths: &HashMap<(u64, u64), String>, +) -> Vec { + let mut unresolved = 0; + let mappings = records + .into_iter() + .filter_map(|record| { + let Some(path) = paths.get(&(record.dev, record.ino)) else { + unresolved += 1; + return None; + }; + + Some(ProcessMapping { + pid: record.pid as i32, + path: path.clone(), + dev: record.dev, + ino: record.ino, + file_offset: record.file_offset, + avma_range: record.start..record.end, + timestamp: record.timestamp, + }) + }) + .collect(); + + if unresolved > 0 { + debug!("{unresolved} mapping records had no resolved path and were dropped"); + } + mappings +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(dev: u64, ino: u64) -> MappingRecord { + MappingRecord { + pid: 5, + dev, + ino, + file_offset: 0x1000, + start: 0x4000, + end: 0x8000, + timestamp: 42, + } + } + + #[test] + fn resolves_records_against_the_path_cache() { + let paths = HashMap::from([((1, 2), "/lib/libc.so.6".to_string())]); + + let mappings = resolve_mappings(vec![record(1, 2)], &paths); + + assert_eq!(mappings.len(), 1); + assert_eq!(mappings[0].path, "/lib/libc.so.6"); + assert_eq!(mappings[0].avma_range, 0x4000..0x8000); + assert_eq!(mappings[0].file_offset, 0x1000); + assert_eq!(mappings[0].pid, 5); + } + + /// A module we cannot name is a module we cannot read, so it must not reach + /// the artifact as an empty path. + #[test] + fn drops_records_without_a_resolved_path() { + assert!(resolve_mappings(vec![record(9, 9)], &HashMap::new()).is_empty()); + } +} diff --git a/crates/memtrack/src/ebpf/mappings/support.rs b/crates/memtrack/src/ebpf/mappings/support.rs new file mode 100644 index 000000000..9e2f3a190 --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/support.rs @@ -0,0 +1,103 @@ +use crate::kernel::KernelVersion; +use crate::prelude::*; + +/// How the running kernel can resolve a mapped file's path inside BPF. +/// +/// Only a BPF LSM program can do it at all: `bpf_d_path()` is restricted to +/// `BPF_TRACE_ITER` programs, sleepable LSM hooks and a fixed fentry allowlist +/// that contains no mmap path (`bpf_d_path_allowed()` in +/// `kernel/trace/bpf_trace.c`), and the `bpf_path_d_path()` kfunc that replaces +/// it rejects every program type but LSM (`bpf_fs_kfuncs_filter()` in +/// `fs/bpf_fs_kfuncs.c`). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum MappingSupport { + /// Paths cannot be resolved, so allocation stacks could not be attributed to + /// modules and are not worth capturing. + Unsupported, + /// Sleepable LSM hook calling `bpf_d_path()` (kernel >= 5.11). + Legacy, + /// LSM hook calling the `bpf_path_d_path()` kfunc (kernel >= 6.12). + Kfunc, +} + +impl MappingSupport { + /// What the running kernel and its boot configuration provide. + /// + /// The kernel release is only half the gate: `bpf` must also be in the + /// active LSM list, which is fixed at boot by `CONFIG_LSM`/`lsm=` and cannot + /// be inferred from the version. + pub fn detect() -> Self { + if !bpf_lsm_active() { + info!( + "The bpf LSM is not active (see /sys/kernel/security/lsm), so mapped module paths \ + cannot be resolved" + ); + return Self::Unsupported; + } + + let version = match KernelVersion::current() { + Ok(version) => version, + Err(e) => { + warn!("Failed to read the kernel version, no mapping records: {e:#}"); + return Self::Unsupported; + } + }; + + let support = Self::for_version(version); + match support { + Self::Unsupported => { + info!("Kernel {version} cannot resolve paths from an LSM program (needs >= 5.11)") + } + Self::Legacy => { + debug!("Kernel {version} predates the bpf_path_d_path kfunc, using bpf_d_path") + } + Self::Kfunc => {} + } + support + } + + fn for_version(version: KernelVersion) -> Self { + if version < KernelVersion::new(5, 11) { + return Self::Unsupported; + } + if version < KernelVersion::new(6, 12) { + return Self::Legacy; + } + Self::Kfunc + } +} + +/// Whether `bpf` is one of the LSMs the running kernel initialized. An +/// unreadable file means securityfs is not mounted, in which case no LSM program +/// will attach either. +fn bpf_lsm_active() -> bool { + const PATH: &str = "/sys/kernel/security/lsm"; + + let Ok(active) = std::fs::read_to_string(PATH) else { + debug!("Could not read {PATH} to check whether the bpf LSM is active"); + return false; + }; + active.trim().split(',').any(|lsm| lsm == "bpf") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `bpf_lsm_mmap_file` has been in the sleepable hook set since 5.11, and + /// 6.12 is the first release carrying `bpf_path_d_path`. + #[test] + fn maps_releases_to_support_levels() { + for (major, minor, expected) in [ + (5, 4, MappingSupport::Unsupported), + (5, 10, MappingSupport::Unsupported), + (5, 11, MappingSupport::Legacy), + (6, 11, MappingSupport::Legacy), + (6, 12, MappingSupport::Kfunc), + (7, 1, MappingSupport::Kfunc), + ] { + let version = KernelVersion::new(major, minor); + assert_eq!(MappingSupport::for_version(version), expected, "{version}"); + } + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index 994650d53..fa8f6f920 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -2,6 +2,7 @@ use super::MemtrackBpf; use crate::ebpf::stacks::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; +use std::collections::HashMap; impl MemtrackBpf { pub fn add_tracked_pid(&mut self, pid: i32) -> Result<()> { @@ -62,6 +63,39 @@ impl MemtrackBpf { ) } + /// Number of mapping records dropped because their ring buffer was full. + /// A non-zero value means a module may be missing from the trace. + pub fn mapping_dropped_count(&self) -> Result { + read_counter( + with_skel!(self, skel => &skel.maps.mapping_dropped), + "mapping_dropped", + ) + } + + /// The paths the kernel resolved for every mapped file, keyed by + /// `(dev, ino)`. Only readable while the BPF object is alive. + pub fn mapped_paths(&self) -> Result> { + let map = with_skel!(self, skel => &skel.maps.path_by_inode); + + let mut paths = HashMap::new(); + for key in map.keys() { + let Some(value) = map + .lookup(&key, libbpf_rs::MapFlags::ANY) + .context("Failed to read a resolved mapping path")? + else { + continue; + }; + + let Some((dev, ino)) = inode_key(&key) else { + continue; + }; + if let Some(path) = inode_path(&value) { + paths.insert((dev, ino), path); + } + } + Ok(paths) + } + pub fn dropped_events_count(&self) -> Result { read_counter( with_skel!(self, skel => &skel.maps.dropped_events), @@ -118,6 +152,24 @@ fn le(bytes: &[u8]) -> u64 { .fold(0, |acc, &b| acc << 8 | u64::from(b)) } +/// Split a `struct inode_key { __u64 dev; __u64 ino; }` map key. +fn inode_key(key: &[u8]) -> Option<(u64, u64)> { + if key.len() < 16 { + return None; + } + Some((le(&key[..8]), le(&key[8..16]))) +} + +/// Read a `struct inode_path { __u32 len; char path[]; }` map value. The kernel +/// wrote `len` bytes including the NUL terminator. +fn inode_path(value: &[u8]) -> Option { + const PATH_OFFSET: usize = 4; + + let len = u32::from_le_bytes(value.get(..PATH_OFFSET)?.try_into().ok()?) as usize; + let path = value.get(PATH_OFFSET..PATH_OFFSET + len.saturating_sub(1))?; + Some(String::from_utf8_lossy(path).into_owned()) +} + /// Read slot 0 of a single-entry `__u64` array map. fn read_counter(map: &impl MapCore, name: &str) -> Result { let key = 0u32; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index cf3014608..7843e4ae9 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; +use crate::ebpf::mappings::MappingSupport; use crate::ebpf::poller::RingBufferPoller; mod token { @@ -27,7 +28,6 @@ pub use rmap::RmapSupport; use crate::bpf_token::has_delegated_bpf_token; use crate::ebpf::TrackerOptions; -use crate::ebpf::stacks::config::clamp_copy_size; /// Which attach mechanism a loaded skeleton uses for its uprobes. See /// `src/ebpf/c/utils/variant.h` for why only one of them is delegatable. @@ -122,13 +122,15 @@ pub struct MemtrackBpf { pub(super) probes: Vec, rmap: RmapSupport, physical: bool, + pub(super) mappings: MappingSupport, } impl MemtrackBpf { /// Load the skeleton, defaulting to the variant a BPF token is available for. /// - /// `options.stack_copy_size` turns on allocation stack capture. - pub fn load(options: TrackerOptions) -> Result { + /// `options.stack_capture` enables allocation stack capture, and `mappings` + /// selects the path-resolving LSM program the running kernel supports. + pub fn load(options: TrackerOptions, mappings: MappingSupport) -> Result { let variant = options.variant.unwrap_or_else(|| { if has_delegated_bpf_token() { BpfVariant::Token @@ -137,7 +139,7 @@ impl MemtrackBpf { } }); let physical = options.physical; - let stack_copy_size = options.stack_copy_size.map(clamp_copy_size); + let capture_stacks = options.stack_capture; crate::kernel::KernelBtf::ensure_available()?; let page_shift = page_shift()?; @@ -166,15 +168,14 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } - if let Some(copy_size) = stack_copy_size { + if capture_stacks { rodata.capture_stacks_enabled = 1; - rodata.stack_copy_size = copy_size; } } // Avoid reserving the stack maps when capture is disabled. A // ring buffer's size must stay a power-of-two page count. - if stack_copy_size.is_none() { + if !capture_stacks { open_skel.maps.stacks.set_max_entries(4096)?; open_skel.maps.stack_traces.set_max_entries(1)?; open_skel.maps.seen_stack_hashes.set_max_entries(1)?; @@ -205,6 +206,26 @@ impl MemtrackBpf { open_skel.progs.tracepoint_rss_stat.set_autoload(false); } + // The kfunc variant fails to load on kernels without + // `bpf_path_d_path`, and neither LSM program can attach when the + // bpf LSM is inactive; without a path there is nothing to + // resolve records against, so the recorder goes too. + match mappings { + MappingSupport::Unsupported => { + open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); + open_skel.progs.cache_mmap_path_legacy.set_autoload(false); + open_skel.progs.record_mmap.set_autoload(false); + open_skel.maps.mappings.set_max_entries(4096)?; + open_skel.maps.path_by_inode.set_max_entries(1)?; + } + MappingSupport::Legacy => { + open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); + } + MappingSupport::Kfunc => { + open_skel.progs.cache_mmap_path_legacy.set_autoload(false); + } + } + $skel(Box::new( open_skel .load() @@ -227,6 +248,7 @@ impl MemtrackBpf { probes: Vec::new(), rmap, physical, + mappings, }) } @@ -292,6 +314,27 @@ impl MemtrackBpf { )) } + /// Poll the mapping-record ring buffer into `tx`. Same contract as + /// [`Self::poll_events_with_channel`]. + pub(crate) fn poll_mappings_with_channel( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.mappings, + crate::ebpf::mappings::MappingRecord::parse, + tx, + poll_interval_ms, + )) + } + + /// Whether the mapping recorder is loaded, i.e. whether its ring buffer is + /// worth polling. + pub fn records_mappings(&self) -> bool { + self.mappings != MappingSupport::Unsupported + } + /// Number of currently-attached probes/links. pub fn probe_count(&self) -> usize { self.probes.len() diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 99a4fc3a0..2c3e5c664 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -1,4 +1,5 @@ use super::{MemtrackBpf, RmapSupport}; +use crate::ebpf::mappings::MappingSupport; use crate::prelude::*; use paste::paste; @@ -66,4 +67,26 @@ impl MemtrackBpf { self.probes.push(link); Ok(()) } + + /// Attach the mapping recorder: the LSM hook caching resolved paths and the + /// `perf_event_mmap` fentry emitting the address records. Only the LSM + /// variant the running kernel supports was loaded. + pub fn attach_mapping_recorder(&mut self) -> Result<()> { + let link = match self.mappings { + MappingSupport::Unsupported => return Ok(()), + MappingSupport::Legacy => { + with_skel!(mut self, skel => skel.progs.cache_mmap_path_legacy.attach()) + } + MappingSupport::Kfunc => { + with_skel!(mut self, skel => skel.progs.cache_mmap_path_kfunc.attach()) + } + } + .context("Failed to attach the mmap path resolver")?; + self.probes.push(link); + + let link = with_skel!(mut self, skel => skel.progs.record_mmap.attach()) + .context("Failed to attach the mapping recorder")?; + self.probes.push(link); + Ok(()) + } } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index d964ebed5..743970c86 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -1,5 +1,6 @@ mod attach_worker; mod events; +pub(crate) mod mappings; mod memtrack; pub(crate) mod poller; mod proc_fs; @@ -7,9 +8,9 @@ mod spawn; mod stacks; mod tracker; +pub use mappings::MappingSupport; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; -pub use stacks::config::{DEFAULT_STACK_COPY_SIZE, clamp_copy_size}; pub use stacks::counters::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 0329cb8d6..f18205f41 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,11 +1,13 @@ use crate::ebpf::attach_worker::AttachWorker; +use crate::ebpf::mappings::{MappingRecord, MappingSupport, resolve_mappings}; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; -use crate::ebpf::stacks::config::stack_copy_size_from_env; +use crate::ebpf::stacks::config::stack_capture_from_env; use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; +use runner_shared::artifacts::MemtrackMappings; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; @@ -26,10 +28,9 @@ pub struct TrackerOptions { /// Uprobe attach mechanism. `None` detects it from BPF token availability. #[builder(default, setter(strip_option))] pub variant: Option, - /// Bytes of user stack to copy for each allocation event. `None` leaves - /// stack capture off; values are clamped to the supported range. - #[builder(default = None)] - pub stack_copy_size: Option, + /// Capture allocation call stacks. + #[builder(default = true)] + pub stack_capture: bool, } impl TrackerOptions { @@ -40,7 +41,7 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .physical(std::env::var("CODSPEED_MEMTRACK_TRACK_PHYSICAL").is_ok_and(|v| v == "1")) - .stack_copy_size(stack_copy_size_from_env()) + .stack_capture(stack_capture_from_env()) .build() } } @@ -52,6 +53,9 @@ pub struct Tracker { /// The dedup gate spans the whole BPF object, so a second session would /// reference stack records the first one already consumed. stacks_polled: Option, + /// Filled by the mapping poller; drained by [`Tracker::mappings`] after the + /// session is dropped, so the poller's final drain is included. + mapping_rx: Mutex>>, } impl Tracker { @@ -62,13 +66,29 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { + let mappings = MappingSupport::detect(); + + // Stacks are raw addresses: without mapped module paths nothing can + // attribute them, so capturing them would only inflate the artifact. + let capture_stacks = match (options.stack_capture, mappings) { + (true, MappingSupport::Unsupported) => { + warn!("Allocation stack capture needs in-kernel path resolution; disabling it"); + false + } + (capture_stacks, _) => capture_stacks, + }; + let options = TrackerOptions { + stack_capture: capture_stacks, + ..options + }; Self::bump_memlock_rlimit()?; - let mut bpf = MemtrackBpf::load(options)?; + let mut bpf = MemtrackBpf::load(options, mappings)?; bpf.attach_tracepoints()?; if options.allocators { bpf.attach_exec_watcher()?; + bpf.attach_mapping_recorder()?; } let bpf = Arc::new(Mutex::new(bpf)); @@ -82,10 +102,8 @@ impl Tracker { bpf, worker: Mutex::new(worker), allocators: options.allocators, - stacks_polled: options - .stack_copy_size - .is_some() - .then(|| AtomicBool::new(false)), + stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), + mapping_rx: Mutex::new(None), }) } @@ -122,17 +140,54 @@ impl Tracker { } let (tx, rx) = mpsc::channel(); - let (poller, stack_poller) = { + let (mapping_tx, mapping_rx) = mpsc::channel(); + let (poller, stack_poller, mapping_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; let stack_poller = capture_stacks .then(|| bpf.poll_stacks(10, tx.clone())) .transpose()?; - (bpf.poll_events_with_channel(10, tx)?, stack_poller) + let mapping_poller = bpf + .records_mappings() + .then(|| bpf.poll_mappings_with_channel(10, mapping_tx)) + .transpose()?; + ( + bpf.poll_events_with_channel(10, tx)?, + stack_poller, + mapping_poller, + ) }; + *self.mapping_rx.lock() = Some(mapping_rx); resume(pid)?; - Ok(Session::new(child, rx, poller, stack_poller)) + Ok(Session::new( + child, + rx, + poller, + stack_poller, + mapping_poller, + )) + } + + /// The module mappings recorded during the run, joined with the paths the + /// kernel resolved for them. Call after dropping the session so the poller's + /// final drain is included, and before the BPF object is torn down. + pub fn mappings(&self) -> Result { + let Some(rx) = self.mapping_rx.lock().take() else { + return Ok(MemtrackMappings::default()); + }; + + let records: Vec<_> = rx.try_iter().collect(); + let paths = self.bpf.lock().mapped_paths()?; + + let dropped = self.bpf.lock().mapping_dropped_count()?; + if dropped > 0 { + warn!("{dropped} mapping records were dropped; some modules may be unresolved"); + } + + Ok(MemtrackMappings { + mappings: resolve_mappings(records, &paths), + }) } /// Enable allocator-event tracking in the BPF program. Lifetime events diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 9bed66b7d..551db639c 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -11,6 +11,7 @@ pub struct Session { events: Option>, _poller: RingBufferPoller, _stack_poller: Option, + _mapping_poller: Option, } impl Session { @@ -19,12 +20,14 @@ impl Session { events: Receiver, poller: RingBufferPoller, stack_poller: Option, + mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, _stack_poller: stack_poller, + _mapping_poller: mapping_poller, } } diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 637217b33..a678bf867 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -242,12 +242,10 @@ pub fn track_command_with_rmap_maps( } /// Track a command with allocation stack capture enabled, returning its events. -pub fn track_command_with_stacks(command: Command, copy_size: u32) -> TrackResult { +pub fn track_command_with_stacks(command: Command) -> TrackResult { track_command_with_opts( command, - TrackerOptions::builder() - .stack_copy_size(Some(copy_size)) - .build(), + TrackerOptions::builder().stack_capture(true).build(), ) } diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 7a95cd6ca..ae3e30f28 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -6,7 +6,7 @@ use std::collections::HashSet; use std::process::Command; use tempfile::TempDir; -const COPY_SIZE: u32 = memtrack::DEFAULT_STACK_COPY_SIZE; +const COPY_SIZE: u32 = 8192; fn compile_fixture( name: &str, @@ -59,8 +59,7 @@ fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box = events .iter() @@ -133,8 +132,7 @@ fn dedup_collapses_repeated_call_paths() -> Result<(), Box Result<(), Box Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_max", &temp_dir)?; - let (events, thread_handle) = - shared::track_command_with_stacks(Command::new(&binary), u32::MAX)?; - - let truncated: Vec<_> = events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record: r } if r.truncated => Some(r.hash), - _ => None, - }) - .collect(); - - assert!( - !record_hashes(&events).is_empty(), - "expected stack records at the maximum copy budget" - ); - assert!( - truncated.is_empty(), - "no capture can be budget-limited at the maximum budget: {truncated:#x?}" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) -} - /// Restores the capture toggle on drop so a failing assertion cannot leak the /// override into later tests (the suite runs single-threaded). struct DisableCaptureGuard; diff --git a/crates/runner-shared/src/artifacts/memtrack/mappings.rs b/crates/runner-shared/src/artifacts/memtrack/mappings.rs new file mode 100644 index 000000000..b271dfee4 --- /dev/null +++ b/crates/runner-shared/src/artifacts/memtrack/mappings.rs @@ -0,0 +1,37 @@ +use libc::pid_t; +use serde::{Deserialize, Serialize}; +use std::ops::Range; + +/// The file-backed mappings the tracked process tree loaded, recorded as they +/// happened. Companion to the event stream: allocation stacks are raw +/// addresses, and these are what turns them back into modules. +/// +/// Kept out of the event stream so a consumer that only needs the module set +/// does not have to decode millions of allocation events. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct MemtrackMappings { + pub mappings: Vec, +} + +impl super::super::ArtifactExt for MemtrackMappings {} + +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessMapping { + pub pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + pub path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + pub dev: u64, + pub ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + pub file_offset: u64, + pub avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + pub timestamp: u64, +} diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 433aab5a5..0ba0ced47 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -2,9 +2,11 @@ use libc::pid_t; use serde::{Deserialize, Serialize}; use std::io::{BufReader, Read, Write}; +mod mappings; mod pipeline; mod writer; +pub use mappings::*; pub use pipeline::*; pub use writer::*; From 138670d20a462ca4b8bbc48d6e861cd6875ef2b9 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:38:28 +0200 Subject: [PATCH 08/21] feat(runner): write memtrack module artifacts and metadata Memory mode now turns the mappings memtrack recorded into the same keyed unwind_data/symbols.map files walltime writes, plus a memtrack.metadata referencing them per pid, so allocation stacks can be unwound off-box. Each mapping's inode is rechecked against the path before its ELF is read: BPF cannot produce a build id, so the recorded (dev, ino) is what proves the file on disk is still the one that was mapped rather than a rebuilt binary whose eh_frame would be bound to the wrong addresses. --- crates/runner-shared/src/metadata.rs | 12 +- src/executor/memory/executor.rs | 46 ++-- src/executor/memory/mod.rs | 1 + src/executor/memory/module_artifacts.rs | 272 ++++++++++++++++++++++++ 4 files changed, 318 insertions(+), 13 deletions(-) create mode 100644 src/executor/memory/module_artifacts.rs diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 654ae298d..2e48a2fd8 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -126,6 +126,16 @@ pub struct MemtrackMetadata { } impl MemtrackMetadata { + pub const CURRENT_VERSION: u64 = 1; + + pub fn new(integration: (String, String), artifacts: ModuleArtifacts) -> Self { + Self { + version: Self::CURRENT_VERSION, + integration, + artifacts, + } + } + pub fn from_reader(reader: R) -> anyhow::Result { serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") } @@ -233,7 +243,7 @@ mod tests { #[test] fn memtrack_metadata_round_trips() { let metadata = MemtrackMetadata { - version: 1, + version: MemtrackMetadata::CURRENT_VERSION, integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), artifacts: populated_artifacts(), }; diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index b8c9a3985..b8a294d8a 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -8,6 +8,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command; use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe_and_callback; use crate::executor::helpers::run_with_env::prefix_command_with_env; use crate::executor::helpers::run_with_sudo::is_root_user; +use crate::executor::memory::module_artifacts::save_module_artifacts; use crate::executor::memory::tunables::MemoryTunables; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, Executor}; @@ -24,6 +25,7 @@ use runner_shared::artifacts::{ArtifactExt, ExecutionTimestamps}; use runner_shared::fifo::Command as FifoCommand; use runner_shared::fifo::IntegrationMode; use semver::Version; +use std::cell::RefCell; use std::fs::canonicalize; use std::path::Path; use std::rc::Rc; @@ -163,7 +165,8 @@ impl Executor for MemoryExecutor { let _tunables = MemoryTunables::apply(); // Create the results/ directory inside the profile folder to avoid having memtrack create it with wrong permissions - std::fs::create_dir_all(execution_context.profile_folder.join("results"))?; + let results_folder = execution_context.profile_folder.join("results"); + std::fs::create_dir_all(&results_folder)?; Self::ensure_privileges()?; @@ -172,16 +175,18 @@ impl Executor for MemoryExecutor { debug!("cmd: {cmd:?}"); let runner_fifo = RunnerFifo::new()?; - let on_process_started = |mut child: std::process::Child| async move { - let (marker_result, exit_status) = - Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + let integration = Rc::new(RefCell::new(None)); + let on_process_started = { + let integration = integration.clone(); + |mut child: std::process::Child| async move { + let (marker_result, fifo_data, exit_status) = + Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + *integration.borrow_mut() = fifo_data.integration; - // Directly write to the profile folder, to avoid having to define another field - marker_result - .save_to(execution_context.profile_folder.join("results")) - .unwrap(); + marker_result.save_to(&results_folder).unwrap(); - Ok(exit_status) + Ok(exit_status) + } }; let status = run_command_with_log_pipe_and_callback(cmd, on_process_started).await?; @@ -191,6 +196,19 @@ impl Executor for MemoryExecutor { bail!("failed to execute memory tracker process: {status}"); } + if let Some(integration) = integration.borrow_mut().take() { + let results_folder = execution_context.profile_folder.join("results"); + if let Err(e) = save_module_artifacts( + &execution_context.profile_folder, + &results_folder, + integration, + ) { + // The memory results are complete without them; only offline + // stack attribution is lost. + error!("Failed to save memtrack module artifacts: {e:#}"); + } + } + Ok(()) } @@ -228,7 +246,11 @@ impl MemoryExecutor { mut runner_fifo: RunnerFifo, ipc: MemtrackIpcServer, child: &mut std::process::Child, - ) -> anyhow::Result<(ExecutionTimestamps, std::process::ExitStatus)> { + ) -> anyhow::Result<( + ExecutionTimestamps, + crate::executor::shared::fifo::FifoBenchmarkData, + std::process::ExitStatus, + )> { // Accept the IPC connection from memtrack and get the sender it sends us // Use a timeout to prevent hanging if the process doesn't start properly // https://github.com/servo/ipc-channel/issues/261 @@ -300,9 +322,9 @@ impl MemoryExecutor { Ok(None) }; - let (marker_result, _, exit_status) = + let (marker_result, fifo_data, exit_status) = runner_fifo.handle_fifo_messages(child, on_cmd).await?; - Ok((marker_result, exit_status)) + Ok((marker_result, fifo_data, exit_status)) } } diff --git a/src/executor/memory/mod.rs b/src/executor/memory/mod.rs index 2d17547d1..9f48a81ab 100644 --- a/src/executor/memory/mod.rs +++ b/src/executor/memory/mod.rs @@ -1,3 +1,4 @@ pub mod executor; +pub(crate) mod module_artifacts; pub(crate) mod setup; pub(crate) mod tunables; diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs new file mode 100644 index 000000000..f8dcd0e9c --- /dev/null +++ b/src/executor/memory/module_artifacts.rs @@ -0,0 +1,272 @@ +use crate::executor::shared::module_artifacts::loaded_module::LoadedModule; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; +use crate::prelude::*; +use runner_shared::artifacts::{ArtifactExt, MemtrackMappings, ProcessMapping}; +use runner_shared::metadata::MemtrackMetadata; +use std::collections::HashMap; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; + +/// Turn the mappings memtrack recorded into the artifacts an offline unwinder +/// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the +/// `memtrack.metadata` referencing them per pid. +/// +/// `results_folder` is where memtrack wrote its artifacts; the keyed files and +/// the metadata land in `profile_folder`, next to walltime's equivalents. +pub fn save_module_artifacts( + profile_folder: &Path, + results_folder: &Path, + integration: (String, String), +) -> Result<()> { + let mappings = read_mappings(results_folder)?; + if mappings.is_empty() { + debug!("No module mappings recorded, skipping memtrack module artifacts"); + return Ok(()); + } + + let loaded_modules = loaded_modules_from_mappings(&mappings); + debug!( + "Extracting artifacts for {} modules from {} mappings", + loaded_modules.len(), + mappings.len() + ); + + let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); + MemtrackMetadata { + version: MEMTRACK_METADATA_CURRENT_VERSION, + integration, + artifacts: saved.artifacts, + } + .save_to(profile_folder) +} + +/// Read every mapping artifact in the folder. One is written per tracked root +/// process, so a run with several of them contributes several files. +fn read_mappings(results_folder: &Path) -> Result> { + let suffix = format!(".{}.msgpack", MemtrackMappings::name()); + + let mut mappings = Vec::new(); + for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { + if !entry.file_name().to_string_lossy().ends_with(&suffix) { + continue; + } + + let file = std::fs::File::open(entry.path())?; + let artifact = MemtrackMappings::decode_from_reader(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?; + mappings.extend(artifact.mappings); + } + Ok(mappings) +} + +fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { + let mut loaded_modules = HashMap::::new(); + + for mapping in mappings { + let path = PathBuf::from(&mapping.path); + if !names_mapped_file(mapping, &path) { + continue; + } + + let load_bias = match ModuleSymbols::compute_load_bias( + &path, + mapping.avma_range.start, + mapping.avma_range.end, + mapping.file_offset, + ) { + Ok(load_bias) => load_bias, + Err(e) => { + debug!("Failed to compute load bias for {}: {e}", mapping.path); + continue; + } + }; + + let loaded_module = loaded_modules.entry(path.clone()).or_default(); + + if loaded_module.module_symbols.is_none() { + match ModuleSymbols::from_elf(&path) { + Ok(symbols) => loaded_module.module_symbols = Some(symbols), + Err(e) => debug!("Failed to load symbols for {}: {e}", mapping.path), + } + } + + // The ELF-derived halves are per file, the mounting is per mapping, so + // only the latter is recomputed for a module mapped more than once. + let unwind_data = match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + process_unwind_data.timestamp = Some(mapping.timestamp); + Some((unwind_data, process_unwind_data)) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } + }; + + let process_loaded_module = loaded_module + .process_loaded_modules + .entry(mapping.pid) + .or_default(); + process_loaded_module.symbols_load_bias = Some(load_bias); + + if let Some((unwind_data, process_unwind_data)) = unwind_data { + loaded_module.unwind_data = Some(unwind_data); + process_loaded_module.process_unwind_data = Some(process_unwind_data); + } + } + + loaded_modules +} + +/// Whether the path still names the file that was mapped. +/// +/// The mapping records the inode the kernel resolved the path from; a file +/// rebuilt or replaced since then is a different inode, and reading unwind data +/// out of it would bind eh_frame from the wrong binary to those addresses. +fn names_mapped_file(mapping: &ProcessMapping, path: &Path) -> bool { + let Ok(metadata) = std::fs::metadata(path) else { + debug!("{} is no longer readable", mapping.path); + return false; + }; + + // The recorded `dev` is the kernel's s_dev encoding, `st_dev` glibc's, so + // only the decomposed major/minor pair is comparable. + let recorded = (mapping.dev >> 20, mapping.dev & 0xF_FFFF, mapping.ino); + let current = ( + u64::from(libc::major(metadata.dev())), + u64::from(libc::minor(metadata.dev())), + metadata.ino(), + ); + + if recorded != current { + debug!( + "{} changed since it was mapped (recorded {recorded:?}, now {current:?})", + mapping.path + ); + return false; + } + true +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { + ProcessMapping { + pid: 42, + path: path.to_string(), + dev, + ino, + file_offset: 0, + avma_range: 0x1000..0x2000, + timestamp: 7, + } + } + + fn s_dev_of(path: &str) -> (u64, u64) { + let metadata = std::fs::metadata(path).unwrap(); + let dev = + u64::from(libc::major(metadata.dev())) << 20 | u64::from(libc::minor(metadata.dev())); + (dev, metadata.ino()) + } + + /// The recorded s_dev encoding and `st_dev` differ, so the check has to + /// decompose both or it rejects every module that did not change. + #[test] + fn accepts_a_file_that_still_has_the_recorded_inode() { + let path = "/proc/self/exe"; + let (dev, ino) = s_dev_of(path); + + assert!(names_mapped_file( + &mapping_for(path, dev, ino), + Path::new(path) + )); + } + + #[test] + fn rejects_a_file_whose_inode_changed() { + let path = "/proc/self/exe"; + let (dev, _) = s_dev_of(path); + + assert!(!names_mapped_file( + &mapping_for(path, dev, 0), + Path::new(path) + )); + } + + #[test] + fn rejects_a_path_that_no_longer_exists() { + let path = "/nonexistent/module.so"; + + assert!(!names_mapped_file( + &mapping_for(path, 1, 2), + Path::new(path) + )); + } + + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_recorded_mapping() { + const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; + + let profile = tempfile::tempdir().unwrap(); + let results = profile.path().join("results"); + std::fs::create_dir_all(&results).unwrap(); + + let (dev, ino) = s_dev_of(MODULE); + MemtrackMappings { + mappings: vec![ProcessMapping { + pid: 1234, + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + avma_range: 0x5555_555a_7000..0x5555_556b_0000, + timestamp: 999, + }], + } + .save_with_pid_to(&results, 1234) + .unwrap(); + + save_module_artifacts( + profile.path(), + &results, + ("codspeed-rust".to_string(), "4.2.0".to_string()), + ) + .unwrap(); + + let metadata = MemtrackMetadata::from_reader( + std::fs::File::open(profile.path().join("memtrack.metadata")).unwrap(), + ) + .unwrap(); + + assert_eq!(metadata.version, MEMTRACK_METADATA_CURRENT_VERSION); + assert_eq!( + metadata.artifacts.mapped_process_module_symbols[&1234].len(), + 1 + ); + + let unwind = &metadata.artifacts.mapped_process_unwind_data_by_pid[&1234][0]; + assert_eq!(unwind.inner.timestamp, Some(999)); + assert!( + profile + .path() + .join(format!("{}.unwind_data", unwind.unwind_data_key)) + .exists() + ); + assert_eq!( + metadata.artifacts.path_key_to_path[&unwind.unwind_data_key], + PathBuf::from(MODULE) + ); + } +} From a7fbd3096ec09da90f56eeb0f267eb8852d232e1 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:38:57 +0200 Subject: [PATCH 09/21] docs(memtrack): describe the mapping recorder --- crates/memtrack/AGENTS.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 0c8d86d41..2439bf0f0 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -20,11 +20,20 @@ Control plane: `src/ipc.rs` exposes an out-of-band `ipc-channel` protocol (`Enab Allocator discovery (`src/allocators/`): `AllocatorLib::find_all()` = dynamic (glob shared libs incl. `/nix/store/*` hints) + static-linked (scan build-dir ELF symbols) + env (`CODSPEED_MEMTRACK_BINARIES`). Each `AllocatorKind` (`Libc`/`LibCpp`/`Jemalloc`/`Mimalloc`/`Tcmalloc`) maps to best-effort attach helpers; only libc must succeed. +Mapping collection (`src/perf_mappings.rs`) uses Linux's native per-CPU perf event stream, not an LSM/BPF availability gate. `PerfMappingPoller` opens a `PERF_TYPE_SOFTWARE` dummy event with `PERF_ATTR_INHERIT` and `PERF_ATTR_MMAP2` on every online CPU for the tracked process, mmaps a perf ring per CPU, and drains those rings on a poll thread. It keeps executable mappings with absolute paths from `PERF_RECORD_MMAP2` and emits the single artifact representation, `MemtrackEventKind::Mapping` inside `MemtrackArtifact.events`, carrying the mapping's pid/tid/timestamp/address/path/device/inode/file offset/length. Opening or enabling any perf event requires the host's perf permissions (for example an allowed `perf_event_paranoid` policy or `CAP_PERFMON`); a permission error is returned from `Tracker::spawn` rather than silently disabling mapping collection. Kernel `PERF_RECORD_LOST` records, ring overruns, and malformed records increment the shared mapping-loss counter. `Tracker::dropped_events_count()` includes that counter with BPF ring-buffer drops, and `codspeed-memtrack track` aborts when the total is non-zero because the artifact is incomplete.` + +### Event stream compatibility + +Session relies on Rust's declaration-order field drop: _poller, _stack_poller, then _perf_mapping_poller. The BPF event and stack pollers therefore disconnect, fully drain, and join before the perf poller is dropped. PerfMappingPoller buffers mapping records and emits them during shutdown, after ordinary allocation/RSS/stack events have reached encode_events; encode_events preserves input order, so Mapping records are a terminal suffix in the one artifact stream. + +This ordering is compatibility-critical. Mapping is a newer event variant; older stream consumers may treat the first unknown Mapping as EOF. Keeping it as the suffix lets those consumers process the complete memory timeline before stopping at that first unknown record. Do not reorder the poller fields or emit mapping records before shutdown. + > Note: the "on-demand attach" design in `.agents/docs/` (AttachWorker, `CODSPEED_MEMTRACK_ONDEMAND`, SIGSTOP/SIGCONT) is a **plan, not yet in source**. Current behavior is upfront attach + `sched_fork` auto-tracking. ## Key Directories -- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `stacks/`, `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/stack_capture.bpf.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/perf_mappings.rs` — native per-CPU `PERF_RECORD_MMAP2` collector. - `src/allocators/` — allocator classification: `mod.rs`, `dynamic.rs`, `static_linked.rs`. - `tests/` — integration tests + `snapshots/` (insta). - `testdata/` — allocation fixtures: `*.c` (gcc), `alloc_cpp/` (cmkr/CMake), `alloc_rust/` + `spawn_wrapper/` (standalone Cargo workspaces). From e59bd3bd594aef5fe0b586fe0d4e1924f2306d52 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 12:40:52 +0200 Subject: [PATCH 10/21] test(memtrack): add nested allocation fixtures --- crates/memtrack/testdata/nested_doubling.c | 44 +++++++++++++++++++ .../testdata/nested_doubling_shared_free.c | 44 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 crates/memtrack/testdata/nested_doubling.c create mode 100644 crates/memtrack/testdata/nested_doubling_shared_free.c diff --git a/crates/memtrack/testdata/nested_doubling.c b/crates/memtrack/testdata/nested_doubling.c new file mode 100644 index 000000000..57badf398 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling.c @@ -0,0 +1,44 @@ +#include +#include + +/* + * Each level allocates twice as much as its caller, then frees on the way back + * up, so the free order is the reverse of the allocation order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(1024) <-- free(2048) <-- free(4096) + * + * Every malloc and every free sits at a distinct call depth, so the six events + * also carry six distinct allocation stacks. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void) { + void* p = malloc(4096); + sleep(1); + escaped_pointer = p; + free(p); +} + +__attribute__((noinline)) static void level2(void) { + void* p = malloc(2048); + escaped_pointer = p; + sleep(1); + level3(); + free(p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + sleep(1); + level2(); + free(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} diff --git a/crates/memtrack/testdata/nested_doubling_shared_free.c b/crates/memtrack/testdata/nested_doubling_shared_free.c new file mode 100644 index 000000000..66c53fd99 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling_shared_free.c @@ -0,0 +1,44 @@ +#include +#include + +/* + * Same doubling allocation chain as nested_doubling.c, but ownership is handed + * down and the innermost level frees all three buffers in reverse order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(4096) + * free(2048) + * free(1024) + * + * The three mallocs come from three different call depths while all three frees + * share one, so a deallocation event must be attributed to the free site rather + * than to wherever its allocation happened. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void* outer, void* middle) { + void* p = malloc(4096); + escaped_pointer = p; + free(p); + free(middle); + free(outer); +} + +__attribute__((noinline)) static void level2(void* outer) { + void* p = malloc(2048); + escaped_pointer = p; + level3(outer, p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + level2(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} From 1f0e80b6b7fba1b54a0a6447888a63a2b44b31c4 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 12:41:18 +0200 Subject: [PATCH 11/21] test(memtrack): assert nested stack identities --- crates/memtrack/tests/stack_tests.rs | 84 ++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index ae3e30f28..28950dd72 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -197,3 +197,87 @@ fn explicit_disable_suppresses_stack_capture() -> Result<(), Box Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + for (name, source) in [ + ( + "nested_doubling", + include_str!("../testdata/nested_doubling.c"), + ), + ( + "nested_doubling_shared_free", + include_str!("../testdata/nested_doubling_shared_free.c"), + ), + ] { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; + + let allocations: Vec<(u64, u64, u64)> = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { size, stack_hash } if (1024..=4096).contains(&size) => { + Some((size, e.addr, stack_hash)) + } + _ => None, + }) + .collect(); + + assert_eq!( + allocations + .iter() + .map(|(size, ..)| *size) + .collect::>(), + vec![1024, 2048, 4096], + "[{name}] each level must allocate twice its caller, outermost first" + ); + + // Both probes of one free report the same address, so dedup by address + // to recover the order the fixture released its buffers in. + let mut released: Vec = Vec::new(); + let mut free_hashes: Vec = Vec::new(); + for event in &events { + let MemtrackEventKind::Free { stack_hash } = event.kind else { + continue; + }; + if released.last() == Some(&event.addr) { + continue; + } + released.push(event.addr); + free_hashes.push(stack_hash); + } + + let allocated: Vec = allocations.iter().map(|(_, addr, _)| *addr).collect(); + let expected: Vec = allocated.iter().rev().copied().collect(); + assert_eq!( + released, expected, + "[{name}] buffers must be freed in the reverse of their allocation order" + ); + + let alloc_hashes: HashSet = allocations.iter().map(|(.., hash)| *hash).collect(); + assert_eq!( + alloc_hashes.len(), + allocations.len(), + "[{name}] each allocation depth must carry its own stack identity" + ); + assert!( + !alloc_hashes.contains(&0) && !free_hashes.contains(&0), + "[{name}] every allocation and free must carry a captured stack" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + } + Ok(()) +} From 92b4ab3fef801c1319e5e8c5e233c6e9d34a1443 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 20:42:02 +0200 Subject: [PATCH 12/21] refactor(memtrack): capture module mappings with perf Replace the BPF LSM path cache and mapping ring with inherited per-CPU PERF_RECORD_MMAP2 collectors. Store executable mappings as a terminal suffix in the main memtrack stream so existing timeline consumers remain compatible, then extract and order them in the runner before generating module artifacts. --- Cargo.lock | 10 + Cargo.toml | 1 + crates/memtrack/AGENTS.md | 2 +- crates/memtrack/Cargo.toml | 1 + crates/memtrack/src/ebpf/c/attach.h | 5 + crates/memtrack/src/ebpf/c/event.h | 25 +- crates/memtrack/src/ebpf/c/main.bpf.c | 1 - crates/memtrack/src/ebpf/c/mappings.bpf.h | 161 ------ crates/memtrack/src/ebpf/mappings/mod.rs | 7 - crates/memtrack/src/ebpf/mappings/records.rs | 83 --- crates/memtrack/src/ebpf/mappings/resolve.rs | 73 --- crates/memtrack/src/ebpf/mappings/support.rs | 103 ---- crates/memtrack/src/ebpf/memtrack/maps.rs | 52 -- crates/memtrack/src/ebpf/memtrack/mod.rs | 60 +-- crates/memtrack/src/ebpf/memtrack/tracking.rs | 23 - crates/memtrack/src/ebpf/mod.rs | 4 +- crates/memtrack/src/ebpf/tracker.rs | 176 +++---- crates/memtrack/src/lib.rs | 2 + crates/memtrack/src/perf_mappings.rs | 437 ++++++++++++++++ crates/memtrack/src/session.rs | 13 +- crates/memtrack/tests/c_tests.rs | 6 +- crates/memtrack/tests/rss_tests.rs | 8 +- crates/memtrack/tests/shared.rs | 57 +-- ...ng_shared_free_stack_capture_disabled.snap | 12 + ...ested_doubling_stack_capture_disabled.snap | 12 + ...__stack_paths_stack_capture_disabled.snap} | 0 crates/memtrack/tests/stack_budget_tests.rs | 22 + crates/memtrack/tests/stack_tests.rs | 360 ++++--------- .../src/artifacts/memtrack/mappings.rs | 37 -- .../src/artifacts/memtrack/mod.rs | 32 +- src/executor/memory/module_artifacts.rs | 479 ++++++++++++++++-- 31 files changed, 1206 insertions(+), 1058 deletions(-) delete mode 100644 crates/memtrack/src/ebpf/c/mappings.bpf.h delete mode 100644 crates/memtrack/src/ebpf/mappings/mod.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/records.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/resolve.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/support.rs create mode 100644 crates/memtrack/src/perf_mappings.rs create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap rename crates/memtrack/tests/snapshots/{stack_tests__stack_capture_disabled.snap => stack_tests__stack_paths_stack_capture_disabled.snap} (100%) create mode 100644 crates/memtrack/tests/stack_budget_tests.rs delete mode 100644 crates/runner-shared/src/artifacts/memtrack/mappings.rs diff --git a/Cargo.lock b/Cargo.lock index 8e555476f..4956c2d3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2402,6 +2402,7 @@ dependencies = [ "object", "parking_lot", "paste", + "perf-event-open-sys", "rayon", "rstest", "runner-shared", @@ -2813,6 +2814,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perf-event-open-sys" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5f8d1487a4ffa23c80a1c355dd27235f9b66fb71ba0f261eb417e4fe8451347" +dependencies = [ + "libc", +] + [[package]] name = "pest" version = "2.8.6" diff --git a/Cargo.toml b/Cargo.toml index 8bd28f039..34be934d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,7 @@ ipc-channel = "0.20" itertools = "0.14.0" rayon = "1.12" linux-perf-event-reader = "0.10.2" # matches the version linux-perf-data resolves to +perf-event-open-sys = "6.0" env_logger = "0.11.10" tempfile = "3.27.0" object = { version = "0.39", default-features = false, features = ["read_core", "elf"] } diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 2439bf0f0..2554c18a6 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -84,7 +84,7 @@ sudo -E cargo test --test c_tests -- --test-threads 1 - **Build toolchain:** `clang` + BTF/vmlinux headers, `libbpf-dev`, `zlib1g-dev`, `pkgconf`, `build-essential`; vendored libbpf also needs `autopoint`/`bison`/`flex`. - `vmlinux.h` is pinned to a specific git rev; `libbpf-rs` uses the `vendored` feature (dist links `libbpf-rs/static`). -Env vars actually wired: `CODSPEED_MEMTRACK_BINARIES` (extra static-allocator binaries), `CODSPEED_LOG` (log filter, default `info`), `SUDO_UID`/`SUDO_GID` (privilege drop), `GITHUB_ACTIONS` (build rebuild trigger + test gate). +Env vars actually wired: `CODSPEED_MEMTRACK_BINARIES` (extra static-allocator binaries), `CODSPEED_MEMTRACK_TRACK_ALLOCATORS` (0/false disables), `CODSPEED_MEMTRACK_TRACK_PHYSICAL` (1 enables), `CODSPEED_MEMTRACK_CAPTURE_STACKS` (1 enables), `CODSPEED_MEMTRACK_STACK_BUDGET` (stack copy size in bytes, default 8192), `CODSPEED_LOG` (log filter, default `info`), `SUDO_UID`/`SUDO_GID` (privilege drop), `GITHUB_ACTIONS` (build rebuild trigger + test gate). ## Testing & QA diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index 4912802da..68da1218f 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -34,6 +34,7 @@ itertools = { workspace = true } paste = "1.0.15" libbpf-rs = { version = "0.26", features = ["vendored"], optional = true } object = { workspace = true } +perf-event-open-sys = { workspace = true } rayon = "1.12" parking_lot = "0.12" typed-builder = "0.23.2" diff --git a/crates/memtrack/src/ebpf/c/attach.h b/crates/memtrack/src/ebpf/c/attach.h index 90cbe4360..e188c7d5f 100644 --- a/crates/memtrack/src/ebpf/c/attach.h +++ b/crates/memtrack/src/ebpf/c/attach.h @@ -14,6 +14,11 @@ #define MEMTRACK_PROT_EXEC 0x4 #define MEMTRACK_SIGSTOP 19 +struct inode_key { + __u64 dev; + __u64 ino; +}; + /* (dev, ino) -> 1; populated by userspace after classify/attach */ BPF_HASH_MAP(known_inodes, struct inode_key, __u8, 8192); /* Requests are 24 B and rare; overflow aborts the run via the counter below */ diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index e3fdab0aa..470e532b0 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -29,7 +29,6 @@ * ip/sp/bp. */ #define MEMTRACK_STACK_REGS 33 -/* Counter slots in the stack_counters array map. */ #define MEMTRACK_STACK_COUNTER_COPY_FAILED 0 #define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1 /* bpf_get_stackid() has several negative outcomes (no user callchain, @@ -43,8 +42,7 @@ struct stack_regs { uint64_t reg[MEMTRACK_STACK_REGS]; }; -/* Head of a stack record; `copy_len` raw stack bytes read upwards from `sp` - * follow it. */ +/* Fixed header followed by `copy_len` bytes read upward from `sp`. */ struct stack_header { uint64_t hash; uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ @@ -115,13 +113,6 @@ struct event { } data; }; -/* Identifies a mapped file across both the attach watcher and the mapping - * recorder. `dev` uses the kernel's s_dev encoding: (major << 20) | minor. */ -struct inode_key { - uint64_t dev; - uint64_t ino; -}; - /* Request from the exec-mapping watcher to the userspace attach worker */ struct attach_request { uint32_t pid; @@ -129,18 +120,4 @@ struct attach_request { uint64_t ino; }; -/* One executable file mapping, mirroring PERF_RECORD_MMAP2. The path is not - * here: it is resolved once per inode into a BPF map that userspace joins - * against, since every mapping of the same file shares it. */ -struct mapping_record { - uint64_t dev; - uint64_t ino; - uint64_t file_offset; /* offset of the mapping's first byte in the file */ - uint64_t start; - uint64_t end; - uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ - uint32_t pid; - uint32_t _pad; -}; - #endif /* __EVENT_H__ */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 7a068c605..b405f572b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,7 +8,6 @@ #include "allocator.h" #include "attach.h" #include "event.h" -#include "mappings.bpf.h" #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" diff --git a/crates/memtrack/src/ebpf/c/mappings.bpf.h b/crates/memtrack/src/ebpf/c/mappings.bpf.h deleted file mode 100644 index 5b5635a12..000000000 --- a/crates/memtrack/src/ebpf/c/mappings.bpf.h +++ /dev/null @@ -1,161 +0,0 @@ -#ifndef __MAPPINGS_BPF_H__ -#define __MAPPINGS_BPF_H__ - -#include "event.h" -#include "utils/folio.h" -#include "utils/map_helpers.h" -#include "utils/process_tracking.h" - -/* == Mapping recorder == - * - * Reconstructs what `PERF_RECORD_MMAP2` gives perf: which file a tracked - * process mapped, where, so raw stack addresses can be attributed to modules - * offline. No single hook carries both halves: - * - * security_mmap_file(file, ..) has the file, runs before the VMA exists - * perf_event_mmap(vma) has the addresses, cannot resolve a path - * - * The path therefore lands in a per-inode cache, and the address-bearing hook - * emits inode-keyed records that userspace joins against that cache while this - * BPF object is still loaded. - * - * Path resolution is only reachable from an LSM program: `bpf_d_path()` is - * restricted to sleepable LSM hooks, `BPF_TRACE_ITER` and an fentry allowlist - * holding no mmap path, and the newer `bpf_path_d_path()` kfunc rejects - * non-LSM program types. Both variants are compiled; userspace autoloads the - * one the running kernel supports and neither when the bpf LSM is inactive. */ - -/* VM_EXEC from linux/mm.h, which vmlinux.h does not carry (it is a macro, not a - * type). Only executable mappings are recorded: unwind data and symbols are - * looked up by text address. */ -#define MEMTRACK_VM_EXEC 0x00000004 - -/* d_path() fails with -ENAMETOOLONG rather than truncating, so a short buffer - * loses whole modules. PATH_MAX keeps that from happening. */ -#define MEMTRACK_MAX_PATH 4096 - -struct inode_path { - __u32 len; /* bytes written by d_path, including the NUL */ - char path[MEMTRACK_MAX_PATH]; -}; - -/* Every mapping of an inode shares its cached path. */ -BPF_HASH_MAP(path_by_inode, struct inode_key, struct inode_path, 2048); - -/* A dropped record may leave a module unresolved. */ -BPF_RINGBUF(mappings, 256 * 1024); -BPF_ARRAY_MAP(mapping_dropped, __u64, 1); - -/* The path does not fit on the BPF stack; build it in this per-CPU scratch map. */ -struct { - __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, struct inode_path); -} path_scratch SEC(".maps"); - -extern int bpf_path_d_path(const struct path* path, char* buf, __u64 buf__sz) __ksym __weak; - -static __always_inline void bump_mapping_dropped(void) { - __u32 zero = 0; - __u64* drops = bpf_map_lookup_elem(&mapping_dropped, &zero); - if (drops) { - __sync_fetch_and_add(drops, 1); - } -} - -/* Return a scratch slot when this inode has no cached path. */ -static __always_inline struct inode_path* mapping_path_slot(struct file* file, - struct inode_key* key) { - if (!file || !is_tracked(current_tgid())) { - return NULL; - } - - key->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); - key->ino = BPF_CORE_READ(file, f_inode, i_ino); - if (bpf_map_lookup_elem(&path_by_inode, key)) { - return NULL; - } - - __u32 zero = 0; - return bpf_map_lookup_elem(&path_scratch, &zero); -} - -/* Publish a resolved path. A failed resolution is not cached, so the next - * mapping of the same inode retries instead of losing the module for the run. */ -static __always_inline void commit_mapping_path(struct inode_key* key, struct inode_path* entry, - int len) { - if (len <= 0) { - return; - } - entry->len = (__u32)len; - bpf_map_update_elem(&path_by_inode, key, entry, BPF_NOEXIST); -} - -/* Kernels >= 6.12: the kfunc is callable from any LSM program. */ -SEC("lsm/mmap_file") -int BPF_PROG(cache_mmap_path_kfunc, struct file* file, unsigned long reqprot, unsigned long prot, - unsigned long flags) { - struct inode_key key = {}; - struct inode_path* entry = mapping_path_slot(file, &key); - if (entry) { - commit_mapping_path(&key, entry, - bpf_path_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); - } - return 0; -} - -/* Kernels 5.11..6.11: `bpf_d_path()` needs a sleepable LSM hook, which - * `mmap_file` has been since 5.11. */ -SEC("lsm.s/mmap_file") -int BPF_PROG(cache_mmap_path_legacy, struct file* file, unsigned long reqprot, unsigned long prot, - unsigned long flags) { - struct inode_key key = {}; - struct inode_path* entry = mapping_path_slot(file, &key); - if (entry) { - commit_mapping_path(&key, entry, bpf_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); - } - return 0; -} - -/* The same hook perf emits MMAP2 from, so the recorded geometry matches what - * the walltime pipeline already consumes: the file offset is in bytes, not - * pages. */ -SEC("fentry/perf_event_mmap") -int BPF_PROG(record_mmap, struct vm_area_struct* vma) { - if (!vma) { - return 0; - } - - __u32 tgid = current_tgid(); - if (!is_tracked(tgid)) { - return 0; - } - - struct file* file = BPF_CORE_READ(vma, vm_file); - if (!file) { - return 0; - } - if (!(BPF_CORE_READ(vma, vm_flags) & MEMTRACK_VM_EXEC)) { - return 0; - } - - struct mapping_record* rec = bpf_ringbuf_reserve(&mappings, sizeof(*rec), 0); - if (!rec) { - bump_mapping_dropped(); - return 0; - } - - rec->pid = tgid; - rec->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); - rec->ino = BPF_CORE_READ(file, f_inode, i_ino); - rec->file_offset = (__u64)BPF_CORE_READ(vma, vm_pgoff) << page_shift; - rec->start = BPF_CORE_READ(vma, vm_start); - rec->end = BPF_CORE_READ(vma, vm_end); - rec->timestamp = bpf_ktime_get_ns(); - bpf_ringbuf_submit(rec, 0); - - return 0; -} - -#endif /* __MAPPINGS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/mappings/mod.rs b/crates/memtrack/src/ebpf/mappings/mod.rs deleted file mode 100644 index 581de916b..000000000 --- a/crates/memtrack/src/ebpf/mappings/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod records; -mod resolve; -mod support; - -pub(crate) use records::MappingRecord; -pub(crate) use resolve::resolve_mappings; -pub use support::MappingSupport; diff --git a/crates/memtrack/src/ebpf/mappings/records.rs b/crates/memtrack/src/ebpf/mappings/records.rs deleted file mode 100644 index 4c660031e..000000000 --- a/crates/memtrack/src/ebpf/mappings/records.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::ebpf::events::bindings::mapping_record; - -/// One executable file mapping as the BPF recorder saw it. The path is resolved -/// separately, per inode. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct MappingRecord { - pub pid: u32, - pub dev: u64, - pub ino: u64, - pub file_offset: u64, - pub start: u64, - pub end: u64, - pub timestamp: u64, -} - -impl MappingRecord { - pub fn parse(data: &[u8]) -> Option { - if data.len() < std::mem::size_of::() { - return None; - } - - // SAFETY: the length is checked above, and the layout is the - // bindgen-generated C ABI struct. - let record: mapping_record = unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; - Some(Self { - pid: record.pid, - dev: record.dev, - ino: record.ino, - file_offset: record.file_offset, - start: record.start, - end: record.end, - timestamp: record.timestamp, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn encode(record: mapping_record) -> Vec { - // SAFETY: reading a plain-data struct as bytes. - unsafe { - std::slice::from_raw_parts( - (&record as *const mapping_record).cast::(), - std::mem::size_of::(), - ) - } - .to_vec() - } - - #[test] - fn well_formed_record_round_trips_every_field() { - let bytes = encode(mapping_record { - dev: 0x1_0002, - ino: 4242, - file_offset: 0x2000, - start: 0x5555_5555_0000, - end: 0x5555_5556_0000, - timestamp: 987_654_321, - pid: 7, - _pad: 0, - }); - - assert_eq!( - MappingRecord::parse(&bytes), - Some(MappingRecord { - pid: 7, - dev: 0x1_0002, - ino: 4242, - file_offset: 0x2000, - start: 0x5555_5555_0000, - end: 0x5555_5556_0000, - timestamp: 987_654_321, - }) - ); - } - - #[test] - fn truncated_buffer_returns_none() { - assert!(MappingRecord::parse(&[0u8; 8]).is_none()); - } -} diff --git a/crates/memtrack/src/ebpf/mappings/resolve.rs b/crates/memtrack/src/ebpf/mappings/resolve.rs deleted file mode 100644 index 9c20ba7f1..000000000 --- a/crates/memtrack/src/ebpf/mappings/resolve.rs +++ /dev/null @@ -1,73 +0,0 @@ -use super::MappingRecord; -use crate::prelude::*; -use runner_shared::artifacts::ProcessMapping; -use std::collections::HashMap; - -/// Records without a path are dropped because their unwind data and symbols cannot be read. -pub(crate) fn resolve_mappings( - records: Vec, - paths: &HashMap<(u64, u64), String>, -) -> Vec { - let mut unresolved = 0; - let mappings = records - .into_iter() - .filter_map(|record| { - let Some(path) = paths.get(&(record.dev, record.ino)) else { - unresolved += 1; - return None; - }; - - Some(ProcessMapping { - pid: record.pid as i32, - path: path.clone(), - dev: record.dev, - ino: record.ino, - file_offset: record.file_offset, - avma_range: record.start..record.end, - timestamp: record.timestamp, - }) - }) - .collect(); - - if unresolved > 0 { - debug!("{unresolved} mapping records had no resolved path and were dropped"); - } - mappings -} - -#[cfg(test)] -mod tests { - use super::*; - - fn record(dev: u64, ino: u64) -> MappingRecord { - MappingRecord { - pid: 5, - dev, - ino, - file_offset: 0x1000, - start: 0x4000, - end: 0x8000, - timestamp: 42, - } - } - - #[test] - fn resolves_records_against_the_path_cache() { - let paths = HashMap::from([((1, 2), "/lib/libc.so.6".to_string())]); - - let mappings = resolve_mappings(vec![record(1, 2)], &paths); - - assert_eq!(mappings.len(), 1); - assert_eq!(mappings[0].path, "/lib/libc.so.6"); - assert_eq!(mappings[0].avma_range, 0x4000..0x8000); - assert_eq!(mappings[0].file_offset, 0x1000); - assert_eq!(mappings[0].pid, 5); - } - - /// A module we cannot name is a module we cannot read, so it must not reach - /// the artifact as an empty path. - #[test] - fn drops_records_without_a_resolved_path() { - assert!(resolve_mappings(vec![record(9, 9)], &HashMap::new()).is_empty()); - } -} diff --git a/crates/memtrack/src/ebpf/mappings/support.rs b/crates/memtrack/src/ebpf/mappings/support.rs deleted file mode 100644 index 9e2f3a190..000000000 --- a/crates/memtrack/src/ebpf/mappings/support.rs +++ /dev/null @@ -1,103 +0,0 @@ -use crate::kernel::KernelVersion; -use crate::prelude::*; - -/// How the running kernel can resolve a mapped file's path inside BPF. -/// -/// Only a BPF LSM program can do it at all: `bpf_d_path()` is restricted to -/// `BPF_TRACE_ITER` programs, sleepable LSM hooks and a fixed fentry allowlist -/// that contains no mmap path (`bpf_d_path_allowed()` in -/// `kernel/trace/bpf_trace.c`), and the `bpf_path_d_path()` kfunc that replaces -/// it rejects every program type but LSM (`bpf_fs_kfuncs_filter()` in -/// `fs/bpf_fs_kfuncs.c`). -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum MappingSupport { - /// Paths cannot be resolved, so allocation stacks could not be attributed to - /// modules and are not worth capturing. - Unsupported, - /// Sleepable LSM hook calling `bpf_d_path()` (kernel >= 5.11). - Legacy, - /// LSM hook calling the `bpf_path_d_path()` kfunc (kernel >= 6.12). - Kfunc, -} - -impl MappingSupport { - /// What the running kernel and its boot configuration provide. - /// - /// The kernel release is only half the gate: `bpf` must also be in the - /// active LSM list, which is fixed at boot by `CONFIG_LSM`/`lsm=` and cannot - /// be inferred from the version. - pub fn detect() -> Self { - if !bpf_lsm_active() { - info!( - "The bpf LSM is not active (see /sys/kernel/security/lsm), so mapped module paths \ - cannot be resolved" - ); - return Self::Unsupported; - } - - let version = match KernelVersion::current() { - Ok(version) => version, - Err(e) => { - warn!("Failed to read the kernel version, no mapping records: {e:#}"); - return Self::Unsupported; - } - }; - - let support = Self::for_version(version); - match support { - Self::Unsupported => { - info!("Kernel {version} cannot resolve paths from an LSM program (needs >= 5.11)") - } - Self::Legacy => { - debug!("Kernel {version} predates the bpf_path_d_path kfunc, using bpf_d_path") - } - Self::Kfunc => {} - } - support - } - - fn for_version(version: KernelVersion) -> Self { - if version < KernelVersion::new(5, 11) { - return Self::Unsupported; - } - if version < KernelVersion::new(6, 12) { - return Self::Legacy; - } - Self::Kfunc - } -} - -/// Whether `bpf` is one of the LSMs the running kernel initialized. An -/// unreadable file means securityfs is not mounted, in which case no LSM program -/// will attach either. -fn bpf_lsm_active() -> bool { - const PATH: &str = "/sys/kernel/security/lsm"; - - let Ok(active) = std::fs::read_to_string(PATH) else { - debug!("Could not read {PATH} to check whether the bpf LSM is active"); - return false; - }; - active.trim().split(',').any(|lsm| lsm == "bpf") -} - -#[cfg(test)] -mod tests { - use super::*; - - /// `bpf_lsm_mmap_file` has been in the sleepable hook set since 5.11, and - /// 6.12 is the first release carrying `bpf_path_d_path`. - #[test] - fn maps_releases_to_support_levels() { - for (major, minor, expected) in [ - (5, 4, MappingSupport::Unsupported), - (5, 10, MappingSupport::Unsupported), - (5, 11, MappingSupport::Legacy), - (6, 11, MappingSupport::Legacy), - (6, 12, MappingSupport::Kfunc), - (7, 1, MappingSupport::Kfunc), - ] { - let version = KernelVersion::new(major, minor); - assert_eq!(MappingSupport::for_version(version), expected, "{version}"); - } - } -} diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index fa8f6f920..994650d53 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -2,7 +2,6 @@ use super::MemtrackBpf; use crate::ebpf::stacks::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; -use std::collections::HashMap; impl MemtrackBpf { pub fn add_tracked_pid(&mut self, pid: i32) -> Result<()> { @@ -63,39 +62,6 @@ impl MemtrackBpf { ) } - /// Number of mapping records dropped because their ring buffer was full. - /// A non-zero value means a module may be missing from the trace. - pub fn mapping_dropped_count(&self) -> Result { - read_counter( - with_skel!(self, skel => &skel.maps.mapping_dropped), - "mapping_dropped", - ) - } - - /// The paths the kernel resolved for every mapped file, keyed by - /// `(dev, ino)`. Only readable while the BPF object is alive. - pub fn mapped_paths(&self) -> Result> { - let map = with_skel!(self, skel => &skel.maps.path_by_inode); - - let mut paths = HashMap::new(); - for key in map.keys() { - let Some(value) = map - .lookup(&key, libbpf_rs::MapFlags::ANY) - .context("Failed to read a resolved mapping path")? - else { - continue; - }; - - let Some((dev, ino)) = inode_key(&key) else { - continue; - }; - if let Some(path) = inode_path(&value) { - paths.insert((dev, ino), path); - } - } - Ok(paths) - } - pub fn dropped_events_count(&self) -> Result { read_counter( with_skel!(self, skel => &skel.maps.dropped_events), @@ -152,24 +118,6 @@ fn le(bytes: &[u8]) -> u64 { .fold(0, |acc, &b| acc << 8 | u64::from(b)) } -/// Split a `struct inode_key { __u64 dev; __u64 ino; }` map key. -fn inode_key(key: &[u8]) -> Option<(u64, u64)> { - if key.len() < 16 { - return None; - } - Some((le(&key[..8]), le(&key[8..16]))) -} - -/// Read a `struct inode_path { __u32 len; char path[]; }` map value. The kernel -/// wrote `len` bytes including the NUL terminator. -fn inode_path(value: &[u8]) -> Option { - const PATH_OFFSET: usize = 4; - - let len = u32::from_le_bytes(value.get(..PATH_OFFSET)?.try_into().ok()?) as usize; - let path = value.get(PATH_OFFSET..PATH_OFFSET + len.saturating_sub(1))?; - Some(String::from_utf8_lossy(path).into_owned()) -} - /// Read slot 0 of a single-entry `__u64` array map. fn read_counter(map: &impl MapCore, name: &str) -> Result { let key = 0u32; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 7843e4ae9..4490fc1e1 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; -use crate::ebpf::mappings::MappingSupport; use crate::ebpf::poller::RingBufferPoller; mod token { @@ -122,15 +121,12 @@ pub struct MemtrackBpf { pub(super) probes: Vec, rmap: RmapSupport, physical: bool, - pub(super) mappings: MappingSupport, } impl MemtrackBpf { - /// Load the skeleton, defaulting to the variant a BPF token is available for. - /// - /// `options.stack_capture` enables allocation stack capture, and `mappings` - /// selects the path-resolving LSM program the running kernel supports. - pub fn load(options: TrackerOptions, mappings: MappingSupport) -> Result { + pub fn new(options: &TrackerOptions) -> Result { + crate::kernel::KernelBtf::ensure_available()?; + let variant = options.variant.unwrap_or_else(|| { if has_delegated_bpf_token() { BpfVariant::Token @@ -140,8 +136,8 @@ impl MemtrackBpf { }); let physical = options.physical; let capture_stacks = options.stack_capture; - crate::kernel::KernelBtf::ensure_available()?; - + let stack_copy_budget = ((options.stack_budget / 512) * 512) + .clamp(512, crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY); let page_shift = page_shift()?; let rmap = if physical { RmapSupport::detect() @@ -170,6 +166,7 @@ impl MemtrackBpf { } if capture_stacks { rodata.capture_stacks_enabled = 1; + rodata.stack_copy_budget = stack_copy_budget; } } @@ -190,7 +187,6 @@ impl MemtrackBpf { } }; } - // Mirrors the attach match in `tracking.rs`. match rmap { RmapSupport::Unsupported => { for_each_rmap_core_prog!(disable_rmap_prog); @@ -206,26 +202,6 @@ impl MemtrackBpf { open_skel.progs.tracepoint_rss_stat.set_autoload(false); } - // The kfunc variant fails to load on kernels without - // `bpf_path_d_path`, and neither LSM program can attach when the - // bpf LSM is inactive; without a path there is nothing to - // resolve records against, so the recorder goes too. - match mappings { - MappingSupport::Unsupported => { - open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); - open_skel.progs.cache_mmap_path_legacy.set_autoload(false); - open_skel.progs.record_mmap.set_autoload(false); - open_skel.maps.mappings.set_max_entries(4096)?; - open_skel.maps.path_by_inode.set_max_entries(1)?; - } - MappingSupport::Legacy => { - open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); - } - MappingSupport::Kfunc => { - open_skel.progs.cache_mmap_path_legacy.set_autoload(false); - } - } - $skel(Box::new( open_skel .load() @@ -248,7 +224,6 @@ impl MemtrackBpf { probes: Vec::new(), rmap, physical, - mappings, }) } @@ -273,7 +248,7 @@ impl MemtrackBpf { poll_interval_ms: u64, tx: std::sync::mpsc::Sender, ) -> Result { - use crate::ebpf::stacks::events; + use crate::ebpf::events; use runner_shared::artifacts::MemtrackEventKind; // The poller outlives this borrow of the skeleton, so the chain lookup @@ -314,27 +289,6 @@ impl MemtrackBpf { )) } - /// Poll the mapping-record ring buffer into `tx`. Same contract as - /// [`Self::poll_events_with_channel`]. - pub(crate) fn poll_mappings_with_channel( - &self, - poll_interval_ms: u64, - tx: std::sync::mpsc::Sender, - ) -> Result { - with_skel!(self, skel => RingBufferPoller::new( - &skel.maps.mappings, - crate::ebpf::mappings::MappingRecord::parse, - tx, - poll_interval_ms, - )) - } - - /// Whether the mapping recorder is loaded, i.e. whether its ring buffer is - /// worth polling. - pub fn records_mappings(&self) -> bool { - self.mappings != MappingSupport::Unsupported - } - /// Number of currently-attached probes/links. pub fn probe_count(&self) -> usize { self.probes.len() diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 2c3e5c664..99a4fc3a0 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -1,5 +1,4 @@ use super::{MemtrackBpf, RmapSupport}; -use crate::ebpf::mappings::MappingSupport; use crate::prelude::*; use paste::paste; @@ -67,26 +66,4 @@ impl MemtrackBpf { self.probes.push(link); Ok(()) } - - /// Attach the mapping recorder: the LSM hook caching resolved paths and the - /// `perf_event_mmap` fentry emitting the address records. Only the LSM - /// variant the running kernel supports was loaded. - pub fn attach_mapping_recorder(&mut self) -> Result<()> { - let link = match self.mappings { - MappingSupport::Unsupported => return Ok(()), - MappingSupport::Legacy => { - with_skel!(mut self, skel => skel.progs.cache_mmap_path_legacy.attach()) - } - MappingSupport::Kfunc => { - with_skel!(mut self, skel => skel.progs.cache_mmap_path_kfunc.attach()) - } - } - .context("Failed to attach the mmap path resolver")?; - self.probes.push(link); - - let link = with_skel!(mut self, skel => skel.progs.record_mmap.attach()) - .context("Failed to attach the mapping recorder")?; - self.probes.push(link); - Ok(()) - } } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 743970c86..6cd589ab9 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -1,6 +1,5 @@ mod attach_worker; mod events; -pub(crate) mod mappings; mod memtrack; pub(crate) mod poller; mod proc_fs; @@ -8,9 +7,8 @@ mod spawn; mod stacks; mod tracker; -pub use mappings::MappingSupport; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; -pub use stacks::counters::StackCaptureStats; +pub use stacks::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index f18205f41..5221fdd0b 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,22 +1,23 @@ use crate::ebpf::attach_worker::AttachWorker; -use crate::ebpf::mappings::{MappingRecord, MappingSupport, resolve_mappings}; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; -use crate::ebpf::stacks::config::stack_capture_from_env; -use crate::ebpf::stacks::counters::StackCaptureStats; +use crate::ebpf::stacks::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; -use runner_shared::artifacts::MemtrackMappings; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; #[derive(Debug, Clone, Copy, TypedBuilder)] pub struct TrackerOptions { + /// BPF attach mechanism, or automatic detection when unset. + #[builder(default)] + pub variant: Option, /// Attach allocator uprobes (malloc/free/calloc/...) through the /// exec-mapping watcher. #[builder(default = true)] @@ -25,12 +26,13 @@ pub struct TrackerOptions { /// folio rmap hooks, which only attach on kernels that expose them. #[builder(default = false)] pub physical: bool, - /// Uprobe attach mechanism. `None` detects it from BPF token availability. - #[builder(default, setter(strip_option))] - pub variant: Option, - /// Capture allocation call stacks. - #[builder(default = true)] + /// Capture allocation call stacks, adding per-allocation stack walking and + /// raw stack copying. + #[builder(default = false)] pub stack_capture: bool, + /// Maximum bytes of user stack to copy per captured call stack. + #[builder(default = 8192)] + pub stack_budget: u32, } impl TrackerOptions { @@ -41,21 +43,37 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .physical(std::env::var("CODSPEED_MEMTRACK_TRACK_PHYSICAL").is_ok_and(|v| v == "1")) - .stack_capture(stack_capture_from_env()) + .stack_capture( + std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").is_ok_and(|v| v == "1"), + ) + .stack_budget( + std::env::var("CODSPEED_MEMTRACK_STACK_BUDGET") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8192), + ) .build() } } +impl Default for TrackerOptions { + fn default() -> Self { + Self::builder().build() + } +} + pub struct Tracker { bpf: Arc>, worker: Mutex>, - allocators: bool, - /// The dedup gate spans the whole BPF object, so a second session would - /// reference stack records the first one already consumed. - stacks_polled: Option, - /// Filled by the mapping poller; drained by [`Tracker::mappings`] after the - /// session is dropped, so the poller's final drain is included. - mapping_rx: Mutex>>, + options: TrackerOptions, + /// Number of native perf mapping records lost due to ring-buffer overflow. + mapping_lost: Arc, +} + +fn kill_and_wait(child: &mut std::process::Child) { + // Cleanup is best effort so the setup error remains the returned error. + let _ = child.kill(); + let _ = child.wait(); } impl Tracker { @@ -66,29 +84,16 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { - let mappings = MappingSupport::detect(); - - // Stacks are raw addresses: without mapped module paths nothing can - // attribute them, so capturing them would only inflate the artifact. - let capture_stacks = match (options.stack_capture, mappings) { - (true, MappingSupport::Unsupported) => { - warn!("Allocation stack capture needs in-kernel path resolution; disabling it"); - false - } - (capture_stacks, _) => capture_stacks, - }; - let options = TrackerOptions { - stack_capture: capture_stacks, - ..options - }; + let bpf = MemtrackBpf::new(&options)?; + Self::build(bpf, options) + } + fn build(mut bpf: MemtrackBpf, options: TrackerOptions) -> Result { Self::bump_memlock_rlimit()?; - let mut bpf = MemtrackBpf::load(options, mappings)?; bpf.attach_tracepoints()?; if options.allocators { bpf.attach_exec_watcher()?; - bpf.attach_mapping_recorder()?; } let bpf = Arc::new(Mutex::new(bpf)); @@ -101,9 +106,8 @@ impl Tracker { Ok(Self { bpf, worker: Mutex::new(worker), - allocators: options.allocators, - stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), - mapping_rx: Mutex::new(None), + options, + mapping_lost: Arc::new(AtomicU64::new(0)), }) } @@ -116,80 +120,60 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { - let capture_stacks = match &self.stacks_polled { - Some(polled) if polled.swap(true, Ordering::Relaxed) => { - bail!("stack capture supports a single spawned command per tracker") - } - Some(_) => true, - None => false, - }; + let capture_stacks = self.options.stack_capture; let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); } - let child = spawn_stopped(&mut wrapped)?; + let mut child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - match self.worker.lock().as_ref() { - Some(worker) => worker.set_root_pid(pid), - // No watcher to arm means exec mappings would be missed. - None if self.allocators => bail!("tracker already finished"), - None => {} - } + let setup = (|| -> Result<_> { + match self.worker.lock().as_ref() { + Some(worker) => worker.set_root_pid(pid), + // No watcher to arm means exec mappings would be missed. + None if self.options.allocators => bail!("tracker already finished"), + None => {} + } - let (tx, rx) = mpsc::channel(); - let (mapping_tx, mapping_rx) = mpsc::channel(); - let (poller, stack_poller, mapping_poller) = { - let mut bpf = self.bpf.lock(); - bpf.add_tracked_pid(pid)?; - let stack_poller = capture_stacks - .then(|| bpf.poll_stacks(10, tx.clone())) - .transpose()?; - let mapping_poller = bpf - .records_mappings() - .then(|| bpf.poll_mappings_with_channel(10, mapping_tx)) + let (tx, rx) = mpsc::channel(); + let (poller, stack_poller) = { + let mut bpf = self.bpf.lock(); + bpf.add_tracked_pid(pid)?; + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + (bpf.poll_events_with_channel(10, tx.clone())?, stack_poller) + }; + let perf_mapping_poller = capture_stacks + .then(|| PerfMappingPoller::start(pid, tx, self.mapping_lost.clone())) .transpose()?; - ( - bpf.poll_events_with_channel(10, tx)?, - stack_poller, - mapping_poller, - ) + + Ok((rx, poller, stack_poller, perf_mapping_poller)) + })(); + let (rx, poller, stack_poller, perf_mapping_poller) = match setup { + Ok(pollers) => pollers, + Err(error) => { + kill_and_wait(&mut child); + return Err(error); + } }; - *self.mapping_rx.lock() = Some(mapping_rx); - resume(pid)?; + + if let Err(error) = resume(pid) { + kill_and_wait(&mut child); + return Err(error); + } Ok(Session::new( child, rx, poller, stack_poller, - mapping_poller, + perf_mapping_poller, )) } - - /// The module mappings recorded during the run, joined with the paths the - /// kernel resolved for them. Call after dropping the session so the poller's - /// final drain is included, and before the BPF object is torn down. - pub fn mappings(&self) -> Result { - let Some(rx) = self.mapping_rx.lock().take() else { - return Ok(MemtrackMappings::default()); - }; - - let records: Vec<_> = rx.try_iter().collect(); - let paths = self.bpf.lock().mapped_paths()?; - - let dropped = self.bpf.lock().mapping_dropped_count()?; - if dropped > 0 { - warn!("{dropped} mapping records were dropped; some modules may be unresolved"); - } - - Ok(MemtrackMappings { - mappings: resolve_mappings(records, &paths), - }) - } - /// Enable allocator-event tracking in the BPF program. Lifetime events /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids /// regardless of this toggle. @@ -205,7 +189,7 @@ impl Tracker { /// Number of events the kernel dropped because the ring buffer was full. /// A non-zero value means the resulting trace is incomplete. pub fn dropped_events_count(&self) -> Result { - self.bpf.lock().dropped_events_count() + Ok(self.bpf.lock().dropped_events_count()? + self.mapping_lost.load(Ordering::Relaxed)) } /// Per-cause counts of stack captures that were skipped or truncated. @@ -213,6 +197,10 @@ impl Tracker { self.bpf.lock().stack_capture_stats() } + pub fn stack_capture_enabled(&self) -> bool { + self.options.stack_capture + } + /// Only meaningful while the BPF object is alive; teardown frees the maps. pub fn ownership_maps(&self) -> Result { self.bpf.lock().ownership_maps() diff --git a/crates/memtrack/src/lib.rs b/crates/memtrack/src/lib.rs index ccd93399f..30cfcc99c 100644 --- a/crates/memtrack/src/lib.rs +++ b/crates/memtrack/src/lib.rs @@ -4,6 +4,8 @@ mod bpf_token; mod ebpf; mod ipc; mod kernel; +#[cfg(feature = "ebpf")] +mod perf_mappings; pub mod prelude; #[cfg(feature = "ebpf")] mod session; diff --git a/crates/memtrack/src/perf_mappings.rs b/crates/memtrack/src/perf_mappings.rs new file mode 100644 index 000000000..45350dbf4 --- /dev/null +++ b/crates/memtrack/src/perf_mappings.rs @@ -0,0 +1,437 @@ +use crate::prelude::*; +use perf_event_open_sys::bindings::{ + PERF_COUNT_SW_DUMMY, PERF_FLAG_FD_CLOEXEC, PERF_RECORD_LOST, PERF_RECORD_MMAP2, + PERF_SAMPLE_TID, PERF_SAMPLE_TIME, PERF_TYPE_SOFTWARE, perf_event_attr, perf_event_header, + perf_event_mmap_page, +}; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::io; +use std::mem::size_of; +use std::os::fd::RawFd; +use std::ptr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::thread::JoinHandle; +use std::time::Duration; + +const DATA_PAGES: usize = 64; + +struct PerfRing { + fd: RawFd, + mapping: *mut u8, + mapping_len: usize, + data_offset: usize, + data_size: usize, + enabled: bool, +} + +// The mapping is exclusively consumed by the poll thread. +unsafe impl Send for PerfRing {} + +impl PerfRing { + fn open(pid: libc::pid_t, cpu: u32, page_size: usize) -> Result { + let mapping_len = page_size + .checked_mul(DATA_PAGES + 1) + .context("perf ring mapping size overflow")?; + ensure!( + mapping_len >= size_of::(), + "perf ring mapping is smaller than its metadata page" + ); + let mut attr = perf_event_attr { + type_: PERF_TYPE_SOFTWARE, + size: size_of::() as u32, + config: PERF_COUNT_SW_DUMMY as u64, + sample_type: (PERF_SAMPLE_TID | PERF_SAMPLE_TIME) as u64, + // PERF_FORMAT_LOST cannot account for inherited child events from this + // parent fd, so PERF_RECORD_LOST remains the complete loss signal. + read_format: 0, + clockid: libc::CLOCK_MONOTONIC, + ..Default::default() + }; + attr.__bindgen_anon_2.wakeup_events = 1; + attr.set_disabled(1); + attr.set_inherit(1); + attr.set_mmap(1); + attr.set_sample_id_all(1); + attr.set_mmap2(1); + attr.set_use_clockid(1); + + let fd = unsafe { + perf_event_open_sys::perf_event_open( + &mut attr, + pid, + cpu as _, + -1, + PERF_FLAG_FD_CLOEXEC as _, + ) + }; + if fd < 0 { + return Err(io::Error::last_os_error()) + .with_context(|| format!("perf_event_open failed for pid {pid} on CPU {cpu}")); + } + + let mapping = unsafe { + libc::mmap( + ptr::null_mut(), + mapping_len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if mapping == libc::MAP_FAILED { + let error = io::Error::last_os_error(); + unsafe { libc::close(fd) }; + return Err(error).context("failed to mmap perf mapping-event ring buffer"); + } + + let page = unsafe { &*(mapping.cast::()) }; + let data_offset = match usize::try_from(page.data_offset) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data offset"); + } + }; + let data_size = match usize::try_from(page.data_size) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data size"); + } + }; + let ring = Self { + fd, + mapping: mapping.cast(), + mapping_len, + data_offset, + data_size, + enabled: false, + }; + + ensure!( + data_offset >= page_size && data_offset % page_size == 0, + "kernel returned an invalid perf ring data offset" + ); + ensure!( + data_size >= size_of::() + && data_size % page_size == 0 + && data_size.is_power_of_two(), + "kernel returned an invalid perf ring data size" + ); + let data_end = data_offset + .checked_add(data_size) + .context("perf ring data range overflow")?; + ensure!( + data_end <= mapping_len, + "kernel returned a perf ring outside the mapped area" + ); + + Ok(ring) + } + + fn enable(&mut self) -> Result<()> { + if unsafe { perf_event_open_sys::ioctls::ENABLE(self.fd, 0) } < 0 { + return Err(io::Error::last_os_error()).context("failed to enable perf mapping events"); + } + self.enabled = true; + Ok(()) + } + + fn drain(&mut self, mappings: &mut Vec, lost: &AtomicU64) { + let page = unsafe { &mut *(self.mapping.cast::()) }; + let head = unsafe { ptr::read_volatile(&page.data_head) }; + std::sync::atomic::fence(Ordering::Acquire); + let mut tail = unsafe { ptr::read_volatile(&page.data_tail) }; + let available = head.wrapping_sub(tail); + + // Once the producer has lapped the consumer, the beginning of the + // stream no longer has a record boundary. Skip the corrupt prefix and + // let the kernel's PERF_RECORD_LOST record account for normal overflow. + if available > self.data_size as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + } else { + while tail != head { + let available = head.wrapping_sub(tail); + if available < size_of::() as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let header = self.copy_from_ring(tail, size_of::()); + let size = u16::from_ne_bytes([header[6], header[7]]) as usize; + if !(size_of::()..=self.data_size).contains(&size) + || size as u64 > available + { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let record = self.copy_from_ring(tail, size); + self.handle_record(&record, mappings, lost); + tail = tail.wrapping_add(size as u64); + } + } + + std::sync::atomic::fence(Ordering::Release); + unsafe { ptr::write_volatile(&mut page.data_tail, tail) }; + } + + fn copy_from_ring(&self, offset: u64, len: usize) -> Vec { + debug_assert!(len <= self.data_size); + let start = offset as usize & (self.data_size - 1); + let first_len = len.min(self.data_size - start); + let data = unsafe { self.mapping.add(self.data_offset) }; + let mut out = Vec::with_capacity(len); + unsafe { + out.extend_from_slice(std::slice::from_raw_parts(data.add(start), first_len)); + if first_len < len { + out.extend_from_slice(std::slice::from_raw_parts(data, len - first_len)); + } + } + out + } + + fn handle_record(&self, record: &[u8], mappings: &mut Vec, lost: &AtomicU64) { + match read_u32(record, 0) { + Some(PERF_RECORD_MMAP2) => { + if let Some(event) = parse_mmap2(record) { + mappings.push(event); + } + } + Some(PERF_RECORD_LOST) => match read_u64(record, 16) { + Some(count) => { + lost.fetch_add(count, Ordering::Relaxed); + } + None => { + lost.fetch_add(1, Ordering::Relaxed); + } + }, + _ => {} + } + } +} + +impl Drop for PerfRing { + fn drop(&mut self) { + unsafe { + if self.enabled { + let _ = perf_event_open_sys::ioctls::DISABLE(self.fd, 0); + } + libc::munmap(self.mapping.cast(), self.mapping_len); + libc::close(self.fd); + } + } +} + +pub(crate) struct PerfMappingPoller { + ctl: Option>>, + thread: Option>, +} + +impl PerfMappingPoller { + pub(crate) fn start( + pid: libc::pid_t, + tx: Sender, + lost: Arc, + ) -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "failed to read the system page size"); + + let cpus = online_cpus()?; + ensure!(!cpus.is_empty(), "no online CPUs reported by the kernel"); + let mut rings = Vec::with_capacity(cpus.len()); + for cpu in cpus { + rings.push(PerfRing::open(pid, cpu, page_size as usize)?); + } + for ring in &mut rings { + ring.enable()?; + } + + let (ctl, ctl_rx) = mpsc::channel::>(); + let thread = std::thread::spawn(move || { + let mut mappings = Vec::new(); + loop { + match ctl_rx.recv_timeout(Duration::from_millis(10)) { + Ok(ack) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + let _ = ack.send(()); + } + Err(RecvTimeoutError::Timeout) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + } + Err(RecvTimeoutError::Disconnected) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + mappings.sort_unstable_by_key(|event| (event.pid, event.timestamp)); + for mapping in mappings { + let _ = tx.send(mapping); + } + break; + } + } + } + }); + + Ok(Self { + ctl: Some(ctl), + thread: Some(thread), + }) + } +} + +impl Drop for PerfMappingPoller { + fn drop(&mut self) { + drop(self.ctl.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn parse_mmap2(record: &[u8]) -> Option { + const FIXED_END: usize = 72; + const SAMPLE_ID_SIZE: usize = 16; + if record.len() < FIXED_END + SAMPLE_ID_SIZE + || read_u32(record, 0)? != PERF_RECORD_MMAP2 + || read_u16(record, 6)? as usize != record.len() + { + return None; + } + + let prot = read_u32(record, 64)?; + if prot & libc::PROT_EXEC as u32 == 0 { + return None; + } + + let path_end = record.len() - SAMPLE_ID_SIZE; + let path_bytes = &record[FIXED_END..path_end]; + let nul = path_bytes.iter().position(|byte| *byte == 0)?; + let path = std::str::from_utf8(&path_bytes[..nul]).ok()?; + if !path.starts_with('/') { + return None; + } + + let major = read_u32(record, 40)? as u64; + let minor = read_u32(record, 44)? as u64; + Some(MemtrackEvent { + pid: read_u32(record, 8)? as libc::pid_t, + tid: read_u32(record, 12)? as libc::pid_t, + timestamp: read_u64(record, record.len() - 8)?, + addr: read_u64(record, 16)?, + kind: MemtrackEventKind::Mapping { + path: path.to_owned(), + dev: (major << 20) | minor, + ino: read_u64(record, 48)?, + file_offset: read_u64(record, 32)?, + len: read_u64(record, 24)?, + }, + }) +} + +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_ne_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_ne_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_u64(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_ne_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +fn online_cpus() -> Result> { + let spec = std::fs::read_to_string("/sys/devices/system/cpu/online") + .context("failed to read online CPUs")?; + parse_cpu_list(spec.trim()) +} + +fn parse_cpu_list(spec: &str) -> Result> { + let mut cpus = Vec::new(); + for part in spec.split(',') { + let part = part.trim(); + ensure!(!part.is_empty(), "invalid empty CPU range"); + let (start, end) = match part.split_once('-') { + Some((start, end)) => (start.parse::()?, end.parse::()?), + None => { + let cpu = part.parse::()?; + (cpu, cpu) + } + }; + ensure!(start <= end, "invalid CPU range {part}"); + cpus.extend(start..=end); + } + Ok(cpus) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_cpu_ranges() { + assert_eq!(parse_cpu_list("0-2,5,8-9").unwrap(), vec![0, 1, 2, 5, 8, 9]); + } + + #[test] + fn parses_executable_mmap2() { + let path = b"/tmp/module.so\0"; + let mut record = vec![0; 72 + path.len() + 16]; + record[0..4].copy_from_slice(&PERF_RECORD_MMAP2.to_ne_bytes()); + let size = record.len() as u16; + record[6..8].copy_from_slice(&size.to_ne_bytes()); + record[8..12].copy_from_slice(&7_u32.to_ne_bytes()); + record[12..16].copy_from_slice(&8_u32.to_ne_bytes()); + record[16..24].copy_from_slice(&0x4000_u64.to_ne_bytes()); + record[24..32].copy_from_slice(&0x2000_u64.to_ne_bytes()); + record[32..40].copy_from_slice(&0x1000_u64.to_ne_bytes()); + record[40..44].copy_from_slice(&1_u32.to_ne_bytes()); + record[44..48].copy_from_slice(&2_u32.to_ne_bytes()); + record[48..56].copy_from_slice(&42_u64.to_ne_bytes()); + record[64..68].copy_from_slice(&(libc::PROT_EXEC as u32).to_ne_bytes()); + record[72..72 + path.len()].copy_from_slice(path); + let timestamp = 99_u64; + let time_offset = record.len() - 8; + record[time_offset..].copy_from_slice(×tamp.to_ne_bytes()); + + assert_eq!( + parse_mmap2(&record), + Some(MemtrackEvent { + pid: 7, + tid: 8, + timestamp, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: "/tmp/module.so".into(), + dev: (1 << 20) | 2, + ino: 42, + file_offset: 0x1000, + len: 0x2000, + }, + }) + ); + } +} diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 551db639c..c8f7ef475 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -1,4 +1,5 @@ use crate::ebpf::poller::RingBufferPoller; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; use std::process::{Child, ExitStatus}; @@ -9,9 +10,15 @@ use std::sync::mpsc::Receiver; pub struct Session { child: Child, events: Option>, + + // Drop order is part of the artifact compatibility contract. Rust drops + // fields in declaration order: both BPF pollers must stay before the perf + // mapping poller. Their Drop implementations disconnect, fully drain, and + // join their poll threads before PerfMappingPoller drops and emits its + // buffered Mapping records as the terminal stream suffix. _poller: RingBufferPoller, _stack_poller: Option, - _mapping_poller: Option, + _perf_mapping_poller: Option, } impl Session { @@ -20,14 +27,14 @@ impl Session { events: Receiver, poller: RingBufferPoller, stack_poller: Option, - mapping_poller: Option, + perf_mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, _stack_poller: stack_poller, - _mapping_poller: mapping_poller, + _perf_mapping_poller: perf_mapping_poller, } } diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index d217ff392..f74de0530 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -2,14 +2,10 @@ mod shared; use rstest::rstest; +use shared::AllocationTestCase; use std::process::Command; use tempfile::TempDir; -struct AllocationTestCase { - name: &'static str, - source: &'static str, -} - const ALLOCATION_TEST_CASES: &[AllocationTestCase] = &[ AllocationTestCase { name: "double_malloc", diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs index 99a787dbe..cdad60e44 100644 --- a/crates/memtrack/tests/rss_tests.rs +++ b/crates/memtrack/tests/rss_tests.rs @@ -358,7 +358,7 @@ fn test_rss_rmap_tracking( #[case] name: &str, ) -> Result<(), Box> { let (raw_report, events) = track_fixture(source, name, |command| { - shared::track_command_with_opts(command, rmap_only_options()) + shared::track_command(command, rmap_only_options()) })?; let raw_report = raw_report.ok_or("fixture wrote no rss report")?; let (rss_stat, rmap) = per_pid_peaks(&events); @@ -442,7 +442,7 @@ fn test_rss_external_reclaim(#[case] mode: Reclaim) -> Result<(), Box Result<(), Box Result<(), Box, std::thread::JoinHandle<()>)>; +pub struct AllocationTestCase { + pub name: &'static str, + pub source: &'static str, +} + /// Snapshot every tracked event, ordered by timestamp and deduplicated by /// `(addr, kind)` so repeated tracking of one allocation counts once. /// @@ -177,7 +182,11 @@ pub fn compile_rust_binary( /// Track a binary, collecting all memory events. pub fn track_binary(binary: &Path) -> TrackResult { - track_command(Command::new(binary)) + track_command(Command::new(binary), None) +} + +pub fn track_binary_with_env(binary: &Path) -> TrackResult { + track_command_with_tracker(Command::new(binary), Tracker::new()?) } pub fn compile_c_source( @@ -201,15 +210,9 @@ pub fn compile_c_source( Ok(binary_path) } -/// Track a command with the default probes: allocators only, discovered by the -/// exec-mapping watcher as the tracked tree maps executables. -pub fn track_command(command: Command) -> TrackResult { - track_command_with_opts(command, TrackerOptions::builder().build()) -} - -/// Track a command under a specific BPF variant rather than the detected one. -pub fn track_command_with_variant(command: Command, variant: BpfVariant) -> TrackResult { - track_command_with_opts(command, TrackerOptions::builder().variant(variant).build()) +pub fn track_command(command: Command, options: impl Into>) -> TrackResult { + let tracker = Tracker::with_options(options.into().unwrap_or_default())?; + track_command_with_tracker(command, tracker) } /// Physical-memory tracking without allocator probes. @@ -220,16 +223,6 @@ fn physical_only_options() -> TrackerOptions { .build() } -/// Track a command with physical-memory tracking enabled. -pub fn track_command_with_rmap(command: Command) -> TrackResult { - track_command_with_opts(command, physical_only_options()) -} - -/// Track a command with an explicit probe selection rather than the environment's. -pub fn track_command_with_opts(command: Command, options: TrackerOptions) -> TrackResult { - track_command_with_tracker(command, Tracker::with_options(options)?) -} - /// Track a command with rmap hooks and snapshot its ownership maps after the /// tracked tree exits but before tracker teardown frees the BPF maps. pub fn track_command_with_rmap_maps( @@ -241,14 +234,6 @@ pub fn track_command_with_rmap_maps( Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } -/// Track a command with allocation stack capture enabled, returning its events. -pub fn track_command_with_stacks(command: Command) -> TrackResult { - track_command_with_opts( - command, - TrackerOptions::builder().stack_capture(true).build(), - ) -} - /// Track a command with rmap hooks and snapshot the ownership maps at a /// fixture-signalled checkpoint. /// @@ -319,14 +304,14 @@ pub fn for_each_variant( let mut profiles: Vec<(BpfVariant, EventProfile)> = Vec::new(); for variant in [BpfVariant::Legacy, BpfVariant::Token] { - let tracker = - match Tracker::with_options(TrackerOptions::builder().variant(variant).build()) { - Ok(tracker) => tracker, - Err(err) => { - eprintln!("skipping {variant:?} variant, cannot attach here: {err:#}"); - continue; - } - }; + let options = TrackerOptions::builder().variant(Some(variant)).build(); + let tracker = match Tracker::with_options(options) { + Ok(tracker) => tracker, + Err(err) => { + eprintln!("skipping {variant:?} variant, cannot attach here: {err:#}"); + continue; + } + }; let (events, thread_handle) = track_command_with_tracker(workload(), tracker)?; assert_events(&events); diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap new file mode 100644 index 000000000..2bd6a37ce --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free_stack_capture_disabled.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: false }", + "Malloc { size: 2048, has_stack: false }", + "Malloc { size: 4096, has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap new file mode 100644 index 000000000..2bd6a37ce --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_stack_capture_disabled.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: false }", + "Malloc { size: 2048, has_stack: false }", + "Malloc { size: 4096, has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_paths_stack_capture_disabled.snap similarity index 100% rename from crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap rename to crates/memtrack/tests/snapshots/stack_tests__stack_paths_stack_capture_disabled.snap diff --git a/crates/memtrack/tests/stack_budget_tests.rs b/crates/memtrack/tests/stack_budget_tests.rs new file mode 100644 index 000000000..d5f80ba9a --- /dev/null +++ b/crates/memtrack/tests/stack_budget_tests.rs @@ -0,0 +1,22 @@ +//! The stack copy budget is a frozen rodata constant, so the verifier's cost of +//! the capture program scales with it. Loading at the default proves nothing +//! about the maximum; both must load. +use memtrack::{BpfVariant, MemtrackBpf, TrackerOptions}; +use rstest::rstest; + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(8192)] +#[case(u32::MAX)] +#[test_log::test] +fn skeleton_loads_at_stack_budget(#[case] budget: u32) { + for variant in [BpfVariant::Legacy, BpfVariant::Token] { + let options = TrackerOptions::builder() + .variant(Some(variant)) + .stack_budget(budget) + .build(); + MemtrackBpf::new(&options).unwrap_or_else(|e| { + panic!("{variant:?} skeleton failed to load at budget {budget}: {e:#}") + }); + } +} diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 28950dd72..bc6ee76b3 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -1,148 +1,108 @@ #[macro_use] mod shared; +use itertools::Itertools; +use memtrack::TrackerOptions; +use rstest::rstest; use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; -use std::collections::HashSet; +use shared::AllocationTestCase; +use std::mem::discriminant; use std::process::Command; use tempfile::TempDir; -const COPY_SIZE: u32 = 8192; - -fn compile_fixture( - name: &str, - temp_dir: &TempDir, -) -> Result> { - shared::compile_c_source( - include_str!("../testdata/stack_paths.c"), - name, - temp_dir.path(), - ) -} -fn require_mapping_support() -> bool { - if memtrack::MappingSupport::detect() == memtrack::MappingSupport::Unsupported { - eprintln!("skipping stack capture test: mapping support is unavailable"); - return false; - } - true -} - -/// The stack identity carried by each allocation and deallocation event that has one. -fn event_hashes(events: &[MemtrackEvent]) -> Vec { - events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { stack_hash, .. } - | MemtrackEventKind::Calloc { stack_hash, .. } - | MemtrackEventKind::AlignedAlloc { stack_hash, .. } - | MemtrackEventKind::Realloc { stack_hash, .. } - | MemtrackEventKind::Free { stack_hash } => (stack_hash != 0).then_some(stack_hash), - _ => None, - }) - .collect() -} +fn describe_allocator_event(kind: &MemtrackEventKind) -> Option { + let description = match kind { + MemtrackEventKind::Malloc { size, stack_hash } => { + format!("Malloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::Calloc { size, stack_hash } => { + format!("Calloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::AlignedAlloc { size, stack_hash } => format!( + "AlignedAlloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ), + MemtrackEventKind::Realloc { + size, stack_hash, .. + } => { + format!( + "Realloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ) + } + MemtrackEventKind::Free { stack_hash } => { + format!("Free {{ has_stack: {} }}", *stack_hash != 0) + } + _ => return None, + }; -fn record_hashes(events: &[MemtrackEvent]) -> HashSet { - events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record } => Some(record.hash), - _ => None, - }) - .collect() + Some(description) } -#[test_with::env(GITHUB_ACTIONS)] -#[test_log::test] -fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths", &temp_dir)?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; - - let records: Vec<_> = events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record: r } => { - Some((r.hash, r.sp, &r.regs, &r.bytes, r.truncated)) - } - _ => None, - }) - .collect(); - - assert!( - records.len() >= 2, - "expected at least two stack records, got {} ({} events)", - records.len(), - events.len() - ); - - let hashes = record_hashes(&events); - assert_eq!( - hashes.len(), - records.len(), - "stack records must be deduplicated by unique hash" - ); - - for (hash, sp, regs, bytes, truncated) in &records { - assert_ne!(*sp, 0, "record {hash:#x} has no stack pointer"); - assert_eq!(regs.len(), 33, "record {hash:#x} must carry 33 registers"); - assert!( - !bytes.is_empty() && bytes.len() % 512 == 0 && bytes.len() <= COPY_SIZE as usize, - "record {hash:#x} must hold whole 512-byte chunks within the budget, got {}", - bytes.len() - ); - assert_eq!( - *truncated, - bytes.len() == COPY_SIZE as usize, - "record {hash:#x} may only be flagged truncated when it filled the budget" - ); - } - - let carried = event_hashes(&events); - assert!( - !carried.is_empty(), - "expected events carrying a captured stack hash" - ); - assert!( - carried.iter().all(|hash| hashes.contains(hash)), - "every non-zero stack_hash must have a matching stack record" - ); - - // The fixture frees every allocation, so both sides must report identities. - assert!( +fn format_events(events: &[MemtrackEvent]) -> Vec { + const MARKER: u64 = 0xC0D5_9EED; + let has_markers = events.iter().any(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { size, .. } if size == MARKER + ) + }); + + let filtered_events = if has_markers { + shared::between_markers(events) + } else { events .iter() - .any(|e| matches!(e.kind, MemtrackEventKind::Free { stack_hash } if stack_hash != 0)), - "free events must carry their own stack identity" - ); + .filter(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { .. } + | MemtrackEventKind::Free { .. } + | MemtrackEventKind::Calloc { .. } + | MemtrackEventKind::Realloc { .. } + | MemtrackEventKind::AlignedAlloc { .. } + ) + }) + .sorted_by_key(|e| e.timestamp) + .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) + .cloned() + .collect() + }; - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) + filtered_events + .iter() + .filter_map(|e| describe_allocator_event(&e.kind)) + .collect() } -#[test_with::env(GITHUB_ACTIONS)] -#[test_log::test] -fn dedup_collapses_repeated_call_paths() -> Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } +const STACK_TEST_CASES: &[AllocationTestCase] = &[ + AllocationTestCase { + name: "stack_paths", + source: include_str!("../testdata/stack_paths.c"), + }, + AllocationTestCase { + name: "nested_doubling", + source: include_str!("../testdata/nested_doubling.c"), + }, + AllocationTestCase { + name: "nested_doubling_shared_free", + source: include_str!("../testdata/nested_doubling_shared_free.c"), + }, +]; + +fn assert_stack_snapshot( + test_case: &AllocationTestCase, + stack_capture: bool, + snapshot_name: &str, +) -> Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; + let binary = shared::compile_c_source(test_case.source, test_case.name, temp_dir.path())?; + let options = TrackerOptions::builder() + .stack_capture(stack_capture) + .build(); + let (events, thread_handle) = shared::track_command(Command::new(binary), options)?; - let carried = event_hashes(&events); - let records = record_hashes(&events); - assert!( - carried.len() > records.len(), - "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", - carried.len(), - records.len(), - events.len() - ); + insta::assert_debug_snapshot!(snapshot_name, format_events(&events)); thread_handle .join() @@ -150,134 +110,30 @@ fn dedup_collapses_repeated_call_paths() -> Result<(), Box Self { - // SAFETY: tests run with --test-threads 1, so no concurrent env access. - unsafe { std::env::set_var("CODSPEED_MEMTRACK_CAPTURE_STACKS", "0") }; - Self - } -} - -impl Drop for DisableCaptureGuard { - fn drop(&mut self) { - // SAFETY: see `set`. - unsafe { std::env::remove_var("CODSPEED_MEMTRACK_CAPTURE_STACKS") }; - } -} - #[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(&STACK_TEST_CASES[0])] +#[case(&STACK_TEST_CASES[1])] +#[case(&STACK_TEST_CASES[2])] #[test_log::test] -fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_disabled", &temp_dir)?; - let _guard = DisableCaptureGuard::set(); - let (events, thread_handle) = shared::track_binary(&binary)?; - - assert!( - events - .iter() - .any(|e| matches!(e.kind, MemtrackEventKind::Malloc { .. })), - "disabled capture must still report allocation events" - ); - assert!( - record_hashes(&events).is_empty(), - "disabled capture must emit zero stack records" - ); - assert!( - event_hashes(&events).is_empty(), - "disabled capture must leave stack_hash zero on every event" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) +fn test_stack_capture( + #[case] test_case: &AllocationTestCase, +) -> Result<(), Box> { + assert_stack_snapshot(test_case, true, test_case.name) } -/// The doubling fixtures allocate down a three-level call chain and free in the -/// reverse order, from three distinct depths (`nested_doubling.c`) or from the -/// innermost frame (`nested_doubling_shared_free.c`). Both must pair every free -/// with its allocation in reverse order and give each allocation depth its own -/// stack identity. #[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case(&STACK_TEST_CASES[0])] +#[case(&STACK_TEST_CASES[1])] +#[case(&STACK_TEST_CASES[2])] #[test_log::test] -fn nested_doubling_frees_in_reverse_order() -> Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } - for (name, source) in [ - ( - "nested_doubling", - include_str!("../testdata/nested_doubling.c"), - ), - ( - "nested_doubling_shared_free", - include_str!("../testdata/nested_doubling_shared_free.c"), - ), - ] { - let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source(source, name, temp_dir.path())?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; - - let allocations: Vec<(u64, u64, u64)> = events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size, stack_hash } if (1024..=4096).contains(&size) => { - Some((size, e.addr, stack_hash)) - } - _ => None, - }) - .collect(); - - assert_eq!( - allocations - .iter() - .map(|(size, ..)| *size) - .collect::>(), - vec![1024, 2048, 4096], - "[{name}] each level must allocate twice its caller, outermost first" - ); - - // Both probes of one free report the same address, so dedup by address - // to recover the order the fixture released its buffers in. - let mut released: Vec = Vec::new(); - let mut free_hashes: Vec = Vec::new(); - for event in &events { - let MemtrackEventKind::Free { stack_hash } = event.kind else { - continue; - }; - if released.last() == Some(&event.addr) { - continue; - } - released.push(event.addr); - free_hashes.push(stack_hash); - } - - let allocated: Vec = allocations.iter().map(|(_, addr, _)| *addr).collect(); - let expected: Vec = allocated.iter().rev().copied().collect(); - assert_eq!( - released, expected, - "[{name}] buffers must be freed in the reverse of their allocation order" - ); - - let alloc_hashes: HashSet = allocations.iter().map(|(.., hash)| *hash).collect(); - assert_eq!( - alloc_hashes.len(), - allocations.len(), - "[{name}] each allocation depth must carry its own stack identity" - ); - assert!( - !alloc_hashes.contains(&0) && !free_hashes.contains(&0), - "[{name}] every allocation and free must carry a captured stack" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - } - Ok(()) +fn test_stack_capture_disabled( + #[case] test_case: &AllocationTestCase, +) -> Result<(), Box> { + assert_stack_snapshot( + test_case, + false, + &format!("{}_stack_capture_disabled", test_case.name), + ) } diff --git a/crates/runner-shared/src/artifacts/memtrack/mappings.rs b/crates/runner-shared/src/artifacts/memtrack/mappings.rs deleted file mode 100644 index b271dfee4..000000000 --- a/crates/runner-shared/src/artifacts/memtrack/mappings.rs +++ /dev/null @@ -1,37 +0,0 @@ -use libc::pid_t; -use serde::{Deserialize, Serialize}; -use std::ops::Range; - -/// The file-backed mappings the tracked process tree loaded, recorded as they -/// happened. Companion to the event stream: allocation stacks are raw -/// addresses, and these are what turns them back into modules. -/// -/// Kept out of the event stream so a consumer that only needs the module set -/// does not have to decode millions of allocation events. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MemtrackMappings { - pub mappings: Vec, -} - -impl super::super::ArtifactExt for MemtrackMappings {} - -/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` -/// would describe it. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessMapping { - pub pid: pid_t, - /// Resolved in-kernel at mmap time, so it is correct for the mapping - /// process's mount namespace even if the process is already gone. - pub path: String, - /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at - /// analysis time that the path still names the file that was mapped. - pub dev: u64, - pub ino: u64, - /// Offset of the mapping's first byte in the file. In bytes, matching - /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. - pub file_offset: u64, - pub avma_range: Range, - /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The - /// mapping is valid from here until a later mapping covers the range. - pub timestamp: u64, -} diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 0ba0ced47..fed115f29 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -2,11 +2,9 @@ use libc::pid_t; use serde::{Deserialize, Serialize}; use std::io::{BufReader, Read, Write}; -mod mappings; mod pipeline; mod writer; -pub use mappings::*; pub use pipeline::*; pub use writer::*; @@ -104,8 +102,18 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + /// One executable file mapping from a native PERF_RECORD_MMAP2 record. + /// The common event header carries its address, process, and timestamp. + Mapping { + path: String, + dev: u64, + ino: u64, + file_offset: u64, + len: u64, + }, Stack { + // Box keeps the MemtrackEventKind enum small across millions of events. #[serde(flatten)] record: Box, }, @@ -180,6 +188,19 @@ mod tests { size: 40960, }, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 400, + addr: 0x400000, + kind: MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, + }, ]; let artifact = MemtrackArtifact { @@ -239,6 +260,13 @@ mod tests { MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, MemtrackEventKind::Stack { record: Box::new(StackRecord { hash: 0xDEAD_BEEF, diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs index f8dcd0e9c..528619d5f 100644 --- a/src/executor/memory/module_artifacts.rs +++ b/src/executor/memory/module_artifacts.rs @@ -3,13 +3,35 @@ use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; -use runner_shared::artifacts::{ArtifactExt, MemtrackMappings, ProcessMapping}; +use libc::pid_t; +use runner_shared::artifacts::{ArtifactExt, MemtrackArtifact, MemtrackEventKind}; use runner_shared::metadata::MemtrackMetadata; +use runner_shared::unwind_data::ProcessUnwindData; use std::collections::HashMap; +use std::ops::Range; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProcessMapping { + pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + dev: u64, + ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + file_offset: u64, + avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + timestamp: u64, +} /// Turn the mappings memtrack recorded into the artifacts an offline unwinder /// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the @@ -36,18 +58,11 @@ pub fn save_module_artifacts( ); let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); - MemtrackMetadata { - version: MEMTRACK_METADATA_CURRENT_VERSION, - integration, - artifacts: saved.artifacts, - } - .save_to(profile_folder) + MemtrackMetadata::new(integration, saved.artifacts).save_to(profile_folder) } -/// Read every mapping artifact in the folder. One is written per tracked root -/// process, so a run with several of them contributes several files. fn read_mappings(results_folder: &Path) -> Result> { - let suffix = format!(".{}.msgpack", MemtrackMappings::name()); + let suffix = format!(".{}.msgpack", MemtrackArtifact::name()); let mut mappings = Vec::new(); for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { @@ -56,10 +71,93 @@ fn read_mappings(results_folder: &Path) -> Result> { } let file = std::fs::File::open(entry.path())?; - let artifact = MemtrackMappings::decode_from_reader(file) - .with_context(|| format!("Failed to decode {:?}", entry.path()))?; - mappings.extend(artifact.mappings); + mappings.extend( + read_mappings_from_artifact(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?, + ); + } + + mappings.sort_unstable_by_key(|mapping| (mapping.pid, mapping.timestamp)); + Ok(mappings) +} + +/// Reconstruct mappings across forks because inherited perf events do not +/// synthesize mappings that already existed when a child was forked. +fn read_mappings_from_artifact(reader: R) -> Result> { + let mut timeline = MemtrackArtifact::decode_streamed(reader)? + .filter(|event| { + matches!( + &event.kind, + MemtrackEventKind::Exec + | MemtrackEventKind::Mapping { .. } + | MemtrackEventKind::Fork { .. } + ) + }) + .collect::>(); + + // Ties break so exec purges before mapping, while fork inherits that mapping. + timeline.sort_by_key(|event| { + let rank = match &event.kind { + MemtrackEventKind::Exec => 0, + MemtrackEventKind::Mapping { .. } => 1, + MemtrackEventKind::Fork { .. } => 2, + _ => unreachable!(), + }; + (event.timestamp, rank) + }); + + let mut live_mappings: HashMap> = HashMap::new(); + let mut mappings = Vec::new(); + + for event in timeline { + match event.kind { + MemtrackEventKind::Mapping { + path, + dev, + ino, + file_offset, + len, + } => { + let Some(end) = event.addr.checked_add(len) else { + debug!("Skipping mapping for {path}: address range overflows"); + continue; + }; + + let mapping = ProcessMapping { + pid: event.pid, + path, + dev, + ino, + file_offset, + avma_range: event.addr..end, + timestamp: event.timestamp, + }; + live_mappings + .entry(mapping.pid) + .or_default() + .push(mapping.clone()); + mappings.push(mapping); + } + MemtrackEventKind::Fork { parent_pid } => { + let inherited = live_mappings.get(&parent_pid).cloned().unwrap_or_default(); + let child_mappings = inherited + .into_iter() + .map(|mut mapping| { + mapping.pid = event.pid; + mapping.timestamp = event.timestamp; + mapping + }) + .collect::>(); + mappings.extend(child_mappings.iter().cloned()); + live_mappings.insert(event.pid, child_mappings); + } + MemtrackEventKind::Exec => { + live_mappings.remove(&event.pid); + } + _ => unreachable!(), + } } + Ok(mappings) } @@ -94,22 +192,29 @@ fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { - process_unwind_data.timestamp = Some(mapping.timestamp); - Some((unwind_data, process_unwind_data)) - } - Err(e) => { - debug!("Failed to load unwind data for {}: {e}", mapping.path); - None + let process_unwind_data = if let Some(unwind_data) = &loaded_module.unwind_data { + Some(ProcessUnwindData { + timestamp: Some(mapping.timestamp), + avma_range: mapping.avma_range.clone(), + base_avma: unwind_data.base_svma.wrapping_add(load_bias), + }) + } else { + match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + loaded_module.unwind_data = Some(unwind_data); + process_unwind_data.timestamp = Some(mapping.timestamp); + Some(process_unwind_data) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } } }; @@ -119,8 +224,7 @@ fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap bool { #[cfg(all(test, target_os = "linux"))] mod tests { use super::*; + use runner_shared::artifacts::MemtrackEvent; fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { ProcessMapping { @@ -216,7 +321,166 @@ mod tests { } #[test] - fn writes_keyed_artifacts_and_metadata_for_a_recorded_mapping() { + fn sorts_interleaved_mapping_artifacts_by_pid_and_timestamp() { + let results = tempfile::tempdir().unwrap(); + + // Separate files model records drained from different per-CPU rings. + MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 20, + addr: 0x2000, + kind: MemtrackEventKind::Mapping { + path: "second-module.so".to_string(), + dev: 2, + ino: 2, + file_offset: 0x2000, + len: 0x1000, + }, + }, + MemtrackEvent { + pid: 7, + tid: 7, + timestamp: 30, + addr: 0x7000, + kind: MemtrackEventKind::Mapping { + path: "child-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0x3000, + len: 0x1000, + }, + }, + ], + } + .save_file_to(results.path(), "cpu1.MemtrackArtifact.msgpack") + .unwrap(); + MemtrackArtifact { + events: vec![MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 10, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: "first-module.so".to_string(), + dev: 1, + ino: 1, + file_offset: 0x1000, + len: 0x1000, + }, + }], + } + .save_file_to(results.path(), "cpu0.MemtrackArtifact.msgpack") + .unwrap(); + + let mappings = read_mappings(results.path()).unwrap(); + assert_eq!( + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.timestamp)) + .collect::>(), + vec![(7, 30), (42, 10), (42, 20)] + ); + assert_eq!(mappings[0].path, "child-module.so"); + assert_eq!(mappings[1].path, "first-module.so"); + assert_eq!(mappings[2].path, "second-module.so"); + } + #[test] + fn extracts_all_mapping_events_from_a_streamed_artifact() { + const FIRST_MODULE: &str = "first-module.so"; + const SECOND_MODULE: &str = "second-module.so"; + let artifact = MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 10, + addr: 0, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, + }, + MemtrackEvent { + pid: 7, + tid: 8, + timestamp: 11, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + len: 0x2000, + }, + }, + MemtrackEvent { + pid: 9, + tid: 10, + timestamp: 12, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + len: 0x3000, + }, + }, + MemtrackEvent { + pid: 99, + tid: 99, + timestamp: 99, + addr: u64::MAX, + kind: MemtrackEventKind::Mapping { + path: "overflow-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0, + len: 1, + }, + }, + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 13, + addr: 0x2000, + kind: MemtrackEventKind::Free { stack_hash: 0 }, + }, + ], + }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + + assert_eq!( + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap(), + vec![ + ProcessMapping { + pid: 7, + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + avma_range: 0x1000..0x3000, + timestamp: 11, + }, + ProcessMapping { + pid: 9, + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + avma_range: 0x4000..0x7000, + timestamp: 12, + }, + ] + ); + } + + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_streamed_mapping() { const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; let profile = tempfile::tempdir().unwrap(); @@ -224,15 +488,19 @@ mod tests { std::fs::create_dir_all(&results).unwrap(); let (dev, ino) = s_dev_of(MODULE); - MemtrackMappings { - mappings: vec![ProcessMapping { + MemtrackArtifact { + events: vec![MemtrackEvent { pid: 1234, - path: MODULE.to_string(), - dev, - ino, - file_offset: 0x5_2000, - avma_range: 0x5555_555a_7000..0x5555_556b_0000, + tid: 1234, timestamp: 999, + addr: 0x5555_555a_7000, + kind: MemtrackEventKind::Mapping { + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + len: 0x109_000, + }, }], } .save_with_pid_to(&results, 1234) @@ -250,7 +518,7 @@ mod tests { ) .unwrap(); - assert_eq!(metadata.version, MEMTRACK_METADATA_CURRENT_VERSION); + assert_eq!(metadata.version, MemtrackMetadata::CURRENT_VERSION); assert_eq!( metadata.artifacts.mapped_process_module_symbols[&1234].len(), 1 @@ -269,4 +537,133 @@ mod tests { PathBuf::from(MODULE) ); } + + fn mapping_event(pid: pid_t, timestamp: u64, addr: u64, path: &str) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr, + kind: MemtrackEventKind::Mapping { + path: path.to_string(), + dev: 1, + ino: 1, + file_offset: 0, + len: 0x1000, + }, + } + } + + fn fork_event(child_pid: pid_t, parent_pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid: child_pid, + tid: child_pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Fork { parent_pid }, + } + } + + fn exec_event(pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Exec, + } + } + + fn decode_lifecycle(events: Vec) -> Vec { + let artifact = MemtrackArtifact { events }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap() + } + + fn mapping_summary(mappings: &[ProcessMapping]) -> Vec<(pid_t, &str, u64)> { + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.path.as_str(), mapping.timestamp)) + .collect() + } + + #[test] + fn fork_without_exec_inherits_only_mappings_before_fork() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + mapping_event(100, 30, 0x2000, "after-fork.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 30), + ] + ); + } + + #[test] + fn exec_stops_inheriting_parent_mappings() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + exec_event(200, 30), + mapping_event(100, 40, 0x2000, "after-fork.so"), + mapping_event(200, 50, 0x3000, "after-exec.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 40), + (200, "after-exec.so", 50), + ] + ); + } + + #[test] + fn grandchild_inherits_transitively_from_forked_child() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "root.so"), + fork_event(200, 100, 20), + mapping_event(200, 25, 0x2000, "child.so"), + fork_event(300, 200, 30), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "root.so", 10), + (200, "root.so", 20), + (200, "child.so", 25), + (300, "root.so", 30), + (300, "child.so", 30), + ] + ); + } + + #[test] + fn equal_timestamp_events_follow_exec_mapping_fork_rank() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-exec.so"), + fork_event(200, 100, 20), + mapping_event(100, 20, 0x2000, "after-exec.so"), + exec_event(100, 20), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-exec.so", 10), + (100, "after-exec.so", 20), + (200, "after-exec.so", 20), + ] + ); + } } From 0b132d9c30fda622aa9b79d64c004007e5f77ea8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 19:43:01 +0200 Subject: [PATCH 13/21] perf(runner-shared): pre-size the frame output buffer Every frame's output buffer started empty and doubled its way to the compressed size, which for a 64k event frame is 8 realloc-and-copy steps over roughly 8 MB. Not measurable in wall clock at current frame sizes; it removes the copy traffic. --- crates/runner-shared/src/artifacts/memtrack/pipeline.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index 8cac46f05..5cc9a10e2 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -13,6 +13,10 @@ const FRAME_EVENTS: usize = 64 * 1024; /// memory to roughly `FRAME_EVENTS * WINDOW_FRAMES` events regardless of how long /// the source runs. const WINDOW_FRAMES: usize = 16; +/// Compressed bytes to reserve per event, so a frame's output buffer does not +/// grow-and-copy its way up from zero. Overshooting wastes a little memory per +/// in-flight frame; undershooting only costs the doublings it fails to avoid. +const FRAME_BYTES_PER_EVENT: usize = 16; /// Encode a stream of events into a single compressed artifact stream, /// compressing frames in parallel across a Rayon pool of `n_workers` threads. @@ -73,7 +77,7 @@ where /// Encode one batch as a single self-contained zstd frame. fn encode_frame(batch: &[MemtrackEvent]) -> anyhow::Result> { - let mut writer = MemtrackWriter::new(Vec::new())?; + let mut writer = MemtrackWriter::new(Vec::with_capacity(batch.len() * FRAME_BYTES_PER_EVENT))?; for event in batch { writer.write_event(event)?; } From d6de4f9cf0fd375503c94a2d988f5b8f87db136a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 19:43:10 +0200 Subject: [PATCH 14/21] ci: benchmark runner-shared in memory mode The memory instrument reports allocation counts and bytes per benchmark, which is what the encode path is bound by. It runs on a hosted runner like simulation does; the runner grants memtrack its capabilities during setup. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3265a9194..d2d363fb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,7 @@ jobs: strategy: fail-fast: false matrix: - mode: [simulation, walltime] + mode: [simulation, walltime, memory] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: From a6a180aacb9a060346746b6eef6731a0bc88b49e Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 3 Sep 2026 12:24:33 +0200 Subject: [PATCH 15/21] refactor(runner-shared): inline pid-key deserializer into metadata --- crates/runner-shared/src/lib.rs | 1 - crates/runner-shared/src/metadata.rs | 32 ++++++++++++++++++-- crates/runner-shared/src/serde_pid_map.rs | 36 ----------------------- src/executor/helpers/debug_file.rs | 29 +++++++++++------- 4 files changed, 47 insertions(+), 51 deletions(-) delete mode 100644 crates/runner-shared/src/serde_pid_map.rs diff --git a/crates/runner-shared/src/lib.rs b/crates/runner-shared/src/lib.rs index 2cdc7d5a6..61e804de7 100644 --- a/crates/runner-shared/src/lib.rs +++ b/crates/runner-shared/src/lib.rs @@ -4,6 +4,5 @@ pub mod fifo; pub mod metadata; pub mod module_symbols; pub mod perf_event; -pub mod serde_pid_map; pub mod unwind_data; pub mod walltime_results; diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 2e48a2fd8..7f41a2088 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -1,5 +1,6 @@ use anyhow::Context; use libc::pid_t; +use serde::de::{Deserializer, Error}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::BufWriter; @@ -11,6 +12,31 @@ use crate::fifo::MarkerType; use crate::module_symbols::MappedProcessModuleSymbols; use crate::unwind_data::MappedProcessUnwindData; +/// Reads a pid-keyed map whose keys arrive as strings. +/// +/// JSON object keys are always strings. serde_json's direct deserializer +/// special-cases that and parses integer map keys, but a `#[serde(flatten)]` +/// field is buffered into serde's internal `Content` first, and that path has +/// no such special case — a `pid_t` key then fails with `invalid type: string`. +/// See . +/// +/// Only the read side needs this: serializing writes the same bytes either way. +fn pid_keys_from_strings<'de, V, D>(deserializer: D) -> Result, D::Error> +where + V: Deserialize<'de>, + D: Deserializer<'de>, +{ + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| { + let pid = key + .parse::() + .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; + Ok((pid, value)) + }) + .collect() +} + /// The per-profile module artifacts: the deduplicated debug info, unwind data /// and symbol tables extracted from the ELF modules the profiled processes /// mapped, plus the per-pid references into them. @@ -28,7 +54,7 @@ pub struct ModuleArtifacts { #[serde( default, skip_serializing_if = "HashMap::is_empty", - with = "crate::serde_pid_map" + deserialize_with = "pid_keys_from_strings" )] pub mapped_process_debug_info_by_pid: HashMap>, @@ -37,7 +63,7 @@ pub struct ModuleArtifacts { #[serde( default, skip_serializing_if = "HashMap::is_empty", - with = "crate::serde_pid_map" + deserialize_with = "pid_keys_from_strings" )] pub mapped_process_unwind_data_by_pid: HashMap>, @@ -46,7 +72,7 @@ pub struct ModuleArtifacts { #[serde( default, skip_serializing_if = "HashMap::is_empty", - with = "crate::serde_pid_map" + deserialize_with = "pid_keys_from_strings" )] pub mapped_process_module_symbols: HashMap>, diff --git a/crates/runner-shared/src/serde_pid_map.rs b/crates/runner-shared/src/serde_pid_map.rs deleted file mode 100644 index fa6fbb9dc..000000000 --- a/crates/runner-shared/src/serde_pid_map.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! `#[serde(with = ...)]` support for pid-keyed maps. -//! -//! JSON object keys are always strings. serde_json's direct deserializer -//! special-cases that and parses integer map keys, but a `#[serde(flatten)]` -//! field is buffered into serde's internal `Content` first, and that path has no -//! such special case — an `i32` key then fails with `invalid type: string`. So -//! the keys are read as strings and parsed here, which works on both paths. - -use libc::pid_t; -use serde::de::{Deserializer, Error}; -use serde::{Deserialize, Serialize, Serializer}; -use std::collections::HashMap; - -pub fn serialize(map: &HashMap, serializer: S) -> Result -where - V: Serialize, - S: Serializer, -{ - map.serialize(serializer) -} - -pub fn deserialize<'de, V, D>(deserializer: D) -> Result, D::Error> -where - V: Deserialize<'de>, - D: Deserializer<'de>, -{ - HashMap::::deserialize(deserializer)? - .into_iter() - .map(|(key, value)| { - let pid = key - .parse::() - .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; - Ok((pid, value)) - }) - .collect() -} diff --git a/src/executor/helpers/debug_file.rs b/src/executor/helpers/debug_file.rs index 0619bed2b..bb1c9d2ba 100644 --- a/src/executor/helpers/debug_file.rs +++ b/src/executor/helpers/debug_file.rs @@ -12,11 +12,17 @@ use std::path::{Path, PathBuf}; /// /// [Separate Debug Files]: https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html pub fn find_debug_file(object: &object::File, binary_path: &Path) -> Option { - ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] + let via_known_roots = ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] .iter() .map(Path::new) .filter(|dir| dir.exists()) - .find_map(|dir| find_debug_file_in(object, binary_path, dir)) + .find_map(|dir| find_debug_file_in(object, binary_path, dir)); + + // `.gnu_debuglink` beside the binary (and its `.debug` subdirectory) is part of + // GDB's search order regardless of whether a global debug-directory root exists + // (see module docs), so it must not be skipped on systems without one, e.g. most + // CI runners. + via_known_roots.or_else(|| find_debug_file_by_debuglink(object, binary_path, None)) } fn find_debug_file_in( @@ -27,7 +33,7 @@ fn find_debug_file_in( if let Some(path) = find_debug_file_by_build_id(object, debug_dir) { return Some(path); } - find_debug_file_by_debuglink(object, binary_path, debug_dir) + find_debug_file_by_debuglink(object, binary_path, Some(debug_dir)) } /// Build-id `a05cfb6313fe06a13c9b4b5cb86c2069faa3951f` resolves to @@ -58,19 +64,20 @@ fn find_debug_file_by_build_id(object: &object::File, debug_dir: &Path) -> Optio fn find_debug_file_by_debuglink( object: &object::File, binary_path: &Path, - debug_dir: &Path, + debug_dir: Option<&Path>, ) -> Option { let (debuglink, expected_crc) = object.gnu_debuglink().ok()??; let debuglink = std::str::from_utf8(debuglink).ok()?; let dir = binary_path.parent()?; - let candidates = [ - dir.join(debuglink), - dir.join(".debug").join(debuglink), - debug_dir - .join(dir.strip_prefix("/").unwrap_or(dir)) - .join(debuglink), - ]; + let mut candidates = vec![dir.join(debuglink), dir.join(".debug").join(debuglink)]; + if let Some(debug_dir) = debug_dir { + candidates.push( + debug_dir + .join(dir.strip_prefix("/").unwrap_or(dir)) + .join(debuglink), + ); + } candidates.into_iter().find(|p| { let Ok(content) = std::fs::read(p) else { From 891423109948527107f6403da29b9667ceac7095 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 3 Sep 2026 17:22:15 +0200 Subject: [PATCH 16/21] fix(memtrack): skip allocator symbols aliased at an attached offset glibc exports cfree at the same file offset as free, so attaching both instrumented one function twice: every free() produced two Free events and two stack captures. Track (library, offset) pairs and skip symbols already covered by an alias. --- crates/memtrack/src/ebpf/memtrack/macros.rs | 22 +++++++++++++++++++++ crates/memtrack/src/ebpf/memtrack/mod.rs | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/crates/memtrack/src/ebpf/memtrack/macros.rs b/crates/memtrack/src/ebpf/memtrack/macros.rs index b1099abe0..dbfcaf276 100644 --- a/crates/memtrack/src/ebpf/memtrack/macros.rs +++ b/crates/memtrack/src/ebpf/memtrack/macros.rs @@ -85,8 +85,19 @@ macro_rules! attach_uprobe_uretprobe { let Some(offset) = symbols.offset(symbol) else { return Ok(false); }; + let key = (lib_path.to_path_buf(), offset); + if self.attached_offsets.contains(&key) { + log::trace!( + "Skipping alias {} at {:#x} in {} (already instrumented)", + symbol, + offset, + lib_path.display() + ); + return Ok(true); + } self.[](lib_path, offset) .with_context(|| format!("Failed to attach {symbol}"))?; + self.attached_offsets.insert(key); log::trace!("Attached {} at {:#x}", symbol, offset); Ok(true) } @@ -121,8 +132,19 @@ macro_rules! attach_uprobe { let Some(offset) = symbols.offset(symbol) else { return Ok(false); }; + let key = (lib_path.to_path_buf(), offset); + if self.attached_offsets.contains(&key) { + log::trace!( + "Skipping alias {} at {:#x} in {} (already instrumented)", + symbol, + offset, + lib_path.display() + ); + return Ok(true); + } self.[](lib_path, offset) .with_context(|| format!("Failed to attach {symbol}"))?; + self.attached_offsets.insert(key); log::trace!("Attached {} at {:#x}", symbol, offset); Ok(true) } diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 4490fc1e1..3ae069b60 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -121,6 +121,11 @@ pub struct MemtrackBpf { pub(super) probes: Vec, rmap: RmapSupport, physical: bool, + /// `(lib_path, offset)` pairs already instrumented. glibc exports + /// symbols like `cfree` (and `free_sized`/`__libc_free` elsewhere) at + /// the same file offset as their canonical function; attaching each + /// alias would double-instrument the one underlying function. + attached_offsets: std::collections::HashSet<(std::path::PathBuf, usize)>, } impl MemtrackBpf { @@ -224,6 +229,7 @@ impl MemtrackBpf { probes: Vec::new(), rmap, physical, + attached_offsets: std::collections::HashSet::new(), }) } From d2177d395d171f900aa15ddb0de0fe2cbf1be6f1 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 4 Sep 2026 10:56:01 +0200 Subject: [PATCH 17/21] perf(memtrack): switch to mimalloc to reduce memory usage and fragmentation mimalloc lowers memtrack's own memory usage, fragmentation and allocation overhead compared to glibc's allocator. As a side effect, it also doesn't route through the exported malloc/free/calloc/realloc symbols, so it skips the allocator uprobes (attached system-wide with pid -1) that would otherwise fire for memtrack's own bookkeeping allocations. Pulled in via the ebpf feature, which the binary already requires. --- Cargo.lock | 19 +++++++++++++++++++ crates/memtrack/Cargo.toml | 3 ++- crates/memtrack/src/main.rs | 3 +++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 4956c2d3b..fee8f75f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2161,6 +2161,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "libredox" version = "0.1.16" @@ -2399,6 +2408,7 @@ dependencies = [ "libbpf-rs", "libc", "log", + "mimalloc", "object", "parking_lot", "paste", @@ -2438,6 +2448,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index 68da1218f..82dccd265 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -16,7 +16,7 @@ required-features = ["ebpf"] [features] default = ["ebpf"] -ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux"] +ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux", "dep:mimalloc"] [dependencies] anyhow = { workspace = true } @@ -38,6 +38,7 @@ perf-event-open-sys = { workspace = true } rayon = "1.12" parking_lot = "0.12" typed-builder = "0.23.2" +mimalloc = { version = "0.1", optional = true } [build-dependencies] libbpf-cargo = { version = "0.26", optional = true } diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index d9a6411b1..64951cf2a 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -1,3 +1,6 @@ +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + use clap::Parser; use ipc_channel::ipc; use memtrack::prelude::*; From 1ba7d2547e8e45d84606c8ed7ae1d7dbe58123a6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 4 Sep 2026 10:56:15 +0200 Subject: [PATCH 18/21] perf(memtrack): resolve stack fp chains off the ring poll thread Each stack record needs its frame-pointer chain looked up in the stack_traces map, which is a syscall per record. Doing that inside the ring-buffer parse callback made the poll thread pay it, so a burst of stacks could push it behind the producer and records were dropped. ResolvingPoller wraps a RingBufferPoller with a dedicated resolver thread: the poll thread only parses (event, stackid) and hands it over an internal channel, and the resolver does the map lookup and forwards the completed event. Drop order keeps the existing shutdown contract, the ring is dropped first so its poll thread joins and closes the internal sender, which lets the resolver drain what it already has before its join returns. --- crates/memtrack/src/ebpf/memtrack/mod.rs | 29 +++++++------ crates/memtrack/src/ebpf/poller.rs | 55 ++++++++++++++++++++++++ crates/memtrack/src/session.rs | 6 +-- 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 3ae069b60..1911e56b1 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; -use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::poller::{RingBufferPoller, ThreadedRingBufferPoller}; mod token { include!(concat!(env!("OUT_DIR"), "/memtrack_token.skel.rs")); @@ -248,33 +248,34 @@ impl MemtrackBpf { )) } - /// Poll the stack-record ring buffer into `tx`. + /// Poll stack records and resolve their frame-pointer chains on a worker thread. + /// Map lookups are syscalls and must not stall the ring-buffer poller. pub(crate) fn poll_stacks( &self, poll_interval_ms: u64, tx: std::sync::mpsc::Sender, - ) -> Result { + ) -> Result { use crate::ebpf::events; use runner_shared::artifacts::MemtrackEventKind; - // The poller outlives this borrow of the skeleton, so the chain lookup - // needs an owned handle rather than a reference to the skeleton map. + // The resolver owns the map handle because it outlives this skeleton borrow. let stack_traces = with_skel!(self, skel => { libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) .context("Failed to create handle for stack_traces map")? }); - let parse = move |data: &[u8]| { - let (mut event, stackid) = events::parse_stack(data)?; - if let MemtrackEventKind::Stack { record } = &mut event.kind { - record.fp_chain = events::fp_chain(&stack_traces, stackid); - } - Some(event) - }; + let resolve = + move |(mut event, stackid): (runner_shared::artifacts::MemtrackEvent, i64)| { + if let MemtrackEventKind::Stack { record } = &mut event.kind { + record.fp_chain = events::fp_chain(&stack_traces, stackid); + } + event + }; - with_skel!(self, skel => RingBufferPoller::new( + with_skel!(self, skel => ThreadedRingBufferPoller::new( &skel.maps.stacks, - parse, + events::parse_stack, + resolve, tx, poll_interval_ms, )) diff --git a/crates/memtrack/src/ebpf/poller.rs b/crates/memtrack/src/ebpf/poller.rs index d80a15549..1e0a81e04 100644 --- a/crates/memtrack/src/ebpf/poller.rs +++ b/crates/memtrack/src/ebpf/poller.rs @@ -77,3 +77,58 @@ impl Drop for RingBufferPoller { } } } + +/// A [`RingBufferPoller`] whose parsed items need a further, potentially +/// expensive step (e.g. a BPF map lookup, which is a syscall) before they are +/// forwarded on `tx`. That step runs on a dedicated resolver thread instead +/// of the poll thread, so a slow per-record resolve can't make the poll +/// thread fall behind the ring and drop records. +pub struct ThreadedRingBufferPoller { + // Drop `ring` first: its poll thread drops the resolver's input sender. + // The resolver then drains parsed items and can be joined safely. + ring: Option, + resolver: Option>, +} + +impl ThreadedRingBufferPoller { + /// Poll `rb_map` with `parse` like [`RingBufferPoller::new`], but run + /// `resolve` on a separate thread: `parse` results are forwarded over an + /// internal channel, and `resolve` turns each one into the value sent on + /// `tx`. + pub fn new( + rb_map: &M, + parse: F, + resolve: R, + tx: Sender, + poll_interval_ms: u64, + ) -> Result + where + M: MapCore, + T: Send + 'static, + U: Send + 'static, + F: Fn(&[u8]) -> Option + Send + 'static, + R: Fn(T) -> U + Send + 'static, + { + let (parsed_tx, parsed_rx) = mpsc::channel::(); + let ring = RingBufferPoller::new(rb_map, parse, parsed_tx, poll_interval_ms)?; + let resolver = std::thread::spawn(move || { + for item in parsed_rx { + let _ = tx.send(resolve(item)); + } + }); + + Ok(Self { + ring: Some(ring), + resolver: Some(resolver), + }) + } +} + +impl Drop for ThreadedRingBufferPoller { + fn drop(&mut self) { + drop(self.ring.take()); + if let Some(resolver) = self.resolver.take() { + let _ = resolver.join(); + } + } +} diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index c8f7ef475..53f02a50a 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -1,4 +1,4 @@ -use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::poller::{RingBufferPoller, ThreadedRingBufferPoller}; use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; @@ -17,7 +17,7 @@ pub struct Session { // join their poll threads before PerfMappingPoller drops and emits its // buffered Mapping records as the terminal stream suffix. _poller: RingBufferPoller, - _stack_poller: Option, + _stack_poller: Option, _perf_mapping_poller: Option, } @@ -26,7 +26,7 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, - stack_poller: Option, + stack_poller: Option, perf_mapping_poller: Option, ) -> Self { Self { From 20f1a15c39aaa26babc79dbb250b18ab540aa41f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 7 Sep 2026 11:45:00 +0200 Subject: [PATCH 19/21] fix(memtrack): read tracking_enabled from a global instead of an array map A single-entry ARRAY map lookup is not actually checked at runtime: for a constant in-range key the verifier strips PTR_MAYBE_NULL and drops the `if (!enabled)` branch as dead code, while the inlined lookup still re-reads the key from the BPF stack. Uprobe programs run under migrate_disable() only, so a program preempting one on the same CPU shares its per-CPU private stack and can clobber that key slot, turning the lookup into an unchecked NULL deref. A global has no key to clobber. This supersedes 97d269c2, whose fail-closed branch the verifier removes anyway. The two toggles became byte-identical apart from the written value, so they now delegate to a shared `set_tracking`. --- .../src/ebpf/c/utils/process_tracking.h | 10 ++---- crates/memtrack/src/ebpf/memtrack/maps.rs | 32 ++++++++----------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/utils/process_tracking.h b/crates/memtrack/src/ebpf/c/utils/process_tracking.h index 4d8444aaf..6c8a893bf 100644 --- a/crates/memtrack/src/ebpf/c/utils/process_tracking.h +++ b/crates/memtrack/src/ebpf/c/utils/process_tracking.h @@ -6,7 +6,7 @@ BPF_HASH_MAP(tracked_pids, __u32, __u8, 10000); BPF_HASH_MAP(pids_ppid, __u32, __u32, 10000); -BPF_ARRAY_MAP(tracking_enabled, __u8, 1); +__u8 tracking_enabled = 0; static __always_inline int is_tracked(__u32 pid) { if (bpf_map_lookup_elem(&tracked_pids, &pid)) { @@ -29,13 +29,7 @@ static __always_inline int is_tracked(__u32 pid) { } static __always_inline int is_enabled(void) { - __u32 key = 0; - __u8* enabled = bpf_map_lookup_elem(&tracking_enabled, &key); - /* ARRAY-map lookups can't fail for a valid index; fail closed if one ever does. */ - if (!enabled) { - return 0; - } - return *enabled; + return tracking_enabled; } static __always_inline void track_child(__u32 child_pid, __u32 parent_pid) { diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index 994650d53..984b228a1 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -16,27 +16,23 @@ impl MemtrackBpf { } pub fn enable_tracking(&mut self) -> Result<()> { - let key = 0u32; - let value = true as u8; - with_skel!(self, skel => skel.maps.tracking_enabled.update( - &key.to_le_bytes(), - &value.to_le_bytes(), - libbpf_rs::MapFlags::ANY, - )) - .context("Failed to enable tracking")?; - Ok(()) + self.set_tracking(true) } pub fn disable_tracking(&mut self) -> Result<()> { - let key = 0u32; - let value = false as u8; - with_skel!(self, skel => skel.maps.tracking_enabled.update( - &key.to_le_bytes(), - &value.to_le_bytes(), - libbpf_rs::MapFlags::ANY, - )) - .context("Failed to disable tracking")?; - Ok(()) + self.set_tracking(false) + } + + fn set_tracking(&mut self, enabled: bool) -> Result<()> { + with_skel!(mut self, skel => { + let bss = skel + .maps + .bss_data + .as_deref_mut() + .context("bss map missing")?; + bss.tracking_enabled = enabled as u8; + Ok(()) + }) } /// Mark a (dev, ino) as classified so the watcher stops re-signalling for it. From 731675efdf9240f4d3b77fb2a4b1b42616156ac3 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 7 Sep 2026 11:46:38 +0200 Subject: [PATCH 20/21] fix(memtrack): detach BPF links through forked fd holders Closing the last reference to a uprobe link fd waits for an RCU-tasks-trace grace period, and hangs indefinitely when the kernel is wedged. Doing that work in this process makes teardown unbounded no matter how many threads share the wait, which is what 705245d4 tried to solve. Fork holder children over disjoint fd chunks instead: they own the terminal close, so this process only drops duplicate references. Holders that do not exit within a shared 30s deadline are abandoned for init to reap, bounding teardown even when the grace period never completes. --- .../memtrack/src/ebpf/memtrack/fd_holder.rs | 401 ++++++++++++++++++ crates/memtrack/src/ebpf/memtrack/mod.rs | 38 +- 2 files changed, 422 insertions(+), 17 deletions(-) create mode 100644 crates/memtrack/src/ebpf/memtrack/fd_holder.rs diff --git a/crates/memtrack/src/ebpf/memtrack/fd_holder.rs b/crates/memtrack/src/ebpf/memtrack/fd_holder.rs new file mode 100644 index 000000000..cd594a581 --- /dev/null +++ b/crates/memtrack/src/ebpf/memtrack/fd_holder.rs @@ -0,0 +1,401 @@ +//! Fork children that hold duplicate fd references, so the terminal close +//! happens in a disposable process rather than here. +//! +//! Closing the last reference to a BPF link fd waits for an RCU-tasks-trace +//! grace period and can hang if the kernel is wedged. [`FdHolderSet`] +//! partitions the fds across children so those waits also run in parallel. + +use std::ops::Range; +use std::os::fd::RawFd; +use std::time::{Duration, Instant}; + +use crate::prelude::*; + +pub struct FdHolder { + child_pid: libc::pid_t, + write_fd: RawFd, +} + +/// Close every open fd except those listed in `keep` (must be sorted +/// ascending, e.g. via `sort_unstable`), by sweeping `close_range` over the +/// gaps between them. `close_range` silently ignores fds that are already +/// closed or out of range, so gaps may safely include fds we never opened. +/// +/// # Safety +/// Only safe to call in the single-threaded child right after `fork()`, +/// before any allocation, locking, or `Drop` impl runs — see +/// [`FdHolder::spawn`]. +unsafe fn close_fds_except(keep: &[RawFd]) { + let mut lo: u32 = 0; + for &fd in keep { + let fd = fd as u32; + if fd > lo { + // SAFETY: caller upholds the fork-child, no-allocation contract. + unsafe { + libc::close_range(lo, fd - 1, 0); + } + } + lo = fd.saturating_add(1); + } + // SAFETY: same as above. + unsafe { + libc::close_range(lo, u32::MAX, 0); + } +} + +impl FdHolder { + /// Fork a child that owns `all_fds[own]` once the caller drops its copies. + /// The child waits for a byte or EOF on a private pipe, then exits. + /// + /// `fork()` duplicates the whole fd table, so the child first closes + /// everything but its chunk and the pipe read end; otherwise it would keep + /// unrelated resources alive. It runs only async-signal-safe libc calls, + /// since forking a multithreaded process leaves locks and allocator state + /// unusable. + pub fn spawn(all_fds: &[RawFd], own: Range) -> std::io::Result { + let mut fds = [0i32; 2]; + // SAFETY: `fds` points to two valid `i32`s, as `pipe(2)` requires. + if unsafe { libc::pipe(fds.as_mut_ptr()) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let [read_fd, write_fd] = fds; + + // Built in the parent, where allocation is still safe. `write_fd` is + // excluded on purpose: the child must give it up, and the sweep closes + // anything absent from this list. + let mut keep: Vec = Vec::with_capacity(all_fds[own.clone()].len() + 1); + keep.push(read_fd); + keep.extend_from_slice(&all_fds[own.clone()]); + keep.sort_unstable(); + + // SAFETY: `fork()` itself is always safe to call; the child branch + // below is restricted to async-signal-safe libc calls until `_exit`. + let pid = unsafe { libc::fork() }; + match pid { + -1 => { + let err = std::io::Error::last_os_error(); + // SAFETY: both fds were just opened by us above. + unsafe { + libc::close(read_fd); + libc::close(write_fd); + } + Err(err) + } + 0 => { + // Keep only the chunk and pipe read end; no Rust code after fork. + unsafe { + close_fds_except(&keep); + let mut buf = [0u8; 1]; + loop { + let n = libc::read(read_fd, buf.as_mut_ptr().cast(), buf.len()); + if n >= 0 { + break; + } + } + libc::_exit(0); + } + } + child_pid => { + // SAFETY: `read_fd` was just opened by us above. + unsafe { + libc::close(read_fd); + } + Ok(Self { + child_pid, + write_fd, + }) + } + } + } + + /// Tell the child to exit. Idempotent, and does not wait for the exit — + /// see [`Self::release`] and [`FdHolderSet::release_all`] for that. + fn signal_release(&mut self) { + if self.write_fd >= 0 { + // SAFETY: `write_fd` is our open pipe write fd. Writing a byte + // ensures the child's `read` returns immediately without + // depending on whether sibling children inherited `write_fd`. + unsafe { + let byte = 0u8; + libc::write(self.write_fd, (&byte as *const u8).cast(), 1); + libc::close(self.write_fd); + } + self.write_fd = -1; + } + } + + /// Signal the child and wait up to `timeout` for it to exit. + /// + /// `false` means the holder is abandoned: the kernel is still tearing down + /// its fds, and init reaps it once that finishes. + pub fn release(mut self, timeout: Duration) -> bool { + self.signal_release(); + + let deadline = Instant::now() + timeout; + loop { + let mut status = 0i32; + // SAFETY: `child_pid` is our own child; `status` is a valid + // out-pointer. `WNOHANG` never blocks. + let ret = unsafe { libc::waitpid(self.child_pid, &mut status, libc::WNOHANG) }; + if ret == self.child_pid { + return true; + } + if ret == -1 { + // ECHILD: nothing left to wait for, already reaped. Any + // other errno (notably EINTR) is transient — fall through + // and retry instead of reporting a false success. + if std::io::Error::last_os_error().raw_os_error() == Some(libc::ECHILD) { + return true; + } + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(20)); + } + } +} + +impl Drop for FdHolder { + fn drop(&mut self) { + self.signal_release(); + } +} + +/// Forked holders for disjoint fd chunks. +pub struct FdHolderSet(Vec); + +impl FdHolderSet { + /// Fork up to `k` holders over roughly equal, contiguous chunks of `fds`. + /// + /// If a fork fails, release holders already created and return an empty set + /// so the caller falls back to direct teardown. + pub fn spawn(fds: &[RawFd], k: usize) -> Self { + if fds.is_empty() { + return Self(Vec::new()); + } + let k = k.clamp(1, fds.len()); + let chunk_len = fds.len().div_ceil(k); + + let mut holders = Vec::with_capacity(k); + for start in (0..fds.len()).step_by(chunk_len) { + let own = start..(start + chunk_len).min(fds.len()); + match FdHolder::spawn(fds, own) { + Ok(holder) => holders.push(holder), + Err(err) => { + debug!( + "Failed to fork fd holder child ({err:#}); falling back to a direct drop for all {} fds", + fds.len() + ); + for holder in holders { + holder.release(Duration::from_secs(5)); + } + return Self(Vec::new()); + } + } + } + Self(holders) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Signal every holder first, then poll them against one shared `timeout`, + /// instead of spending a separate budget on each. + /// + /// `false` means at least one holder is abandoned: the kernel is still + /// tearing down its fds, and init reaps them once that finishes. + pub fn release_all(mut self, timeout: Duration) -> bool { + for holder in &mut self.0 { + holder.signal_release(); + } + + let mut pending: Vec = self.0.iter().map(|holder| holder.child_pid).collect(); + let deadline = Instant::now() + timeout; + loop { + pending.retain(|&pid| { + let mut status = 0i32; + // SAFETY: `pid` is one of our own children; `status` is a + // valid out-pointer. `WNOHANG` never blocks. + let ret = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + if ret == pid { + return false; + } + if ret == -1 { + // ECHILD: nothing left to wait for. Any other errno + // (notably EINTR) is transient; keep polling instead of + // treating it as a reap. + return std::io::Error::last_os_error().raw_os_error() != Some(libc::ECHILD); + } + true + }); + if pending.is_empty() { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(20)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn holder_exits_only_after_release() { + let holder = FdHolder::spawn(&[], 0..0).unwrap(); + let pid = holder.child_pid; + + let mut status = 0i32; + // SAFETY: `pid` is our own child; `status` is a valid out-pointer. + let ret = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + assert_eq!(ret, 0, "holder exited before release() was called"); + + assert!( + holder.release(Duration::from_secs(5)), + "holder did not exit within the timeout after release()" + ); + + // SAFETY: same as above. + let ret = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + assert_eq!(ret, -1, "child pid was still waitable after being reaped"); + } + + /// Set a pipe read end non-blocking so a `read` on it reports "no data + /// yet" (`EAGAIN`) rather than blocking, letting the test distinguish + /// that from EOF (`read` returning `0`). + fn set_nonblocking(fd: RawFd) { + // SAFETY: `fd` is a valid, open fd owned by the caller. + unsafe { + let flags = libc::fcntl(fd, libc::F_GETFL); + libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK); + } + } + + /// `true` if `fd` currently reports EOF (every writer closed), `false` + /// if it reports "no data yet" (at least one writer still open). + fn is_eof(fd: RawFd) -> bool { + let mut buf = [0u8; 1]; + // SAFETY: `fd` is a valid, open, non-blocking pipe read end; `buf` + // is a valid 1-byte out-buffer. + let n = unsafe { libc::read(fd, buf.as_mut_ptr().cast(), buf.len()) }; + if n == 0 { + return true; + } + assert_eq!( + n, -1, + "expected EAGAIN (no data) or EOF (0), got {n} bytes of unexpected data" + ); + let errno = std::io::Error::last_os_error(); + assert_eq!( + errno.raw_os_error(), + Some(libc::EAGAIN), + "unexpected read error: {errno}" + ); + false + } + + /// Poll `fd` for EOF for up to `timeout`, to avoid a race between a + /// just-`fork`ed child's close sweep and this process's own check. + fn wait_for_eof(fd: RawFd, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + if is_eof(fd) { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(5)); + } + } + + /// The fd-ownership contract every consumer relies on: after a chunk's + /// fds are handed to a holder and the caller closes its own copies, + /// releasing one holder tears down only *its* chunk, leaving fds owned + /// by other holders untouched. + #[test] + fn release_only_tears_down_its_own_chunk() { + let mut pipe_a = [0i32; 2]; + let mut pipe_b = [0i32; 2]; + // SAFETY: both arrays point to two valid `i32`s, as `pipe(2)` + // requires. + unsafe { + assert_eq!(libc::pipe(pipe_a.as_mut_ptr()), 0); + assert_eq!(libc::pipe(pipe_b.as_mut_ptr()), 0); + } + let [read_a, write_a] = pipe_a; + let [read_b, write_b] = pipe_b; + set_nonblocking(read_a); + set_nonblocking(read_b); + + let fds = [write_a, write_b]; + let holder_a = FdHolder::spawn(&fds, 0..1).unwrap(); + let holder_b = FdHolder::spawn(&fds, 1..2).unwrap(); + + // SAFETY: both fds were just opened by us above. + unsafe { + libc::close(write_a); + libc::close(write_b); + } + + assert!(!is_eof(read_a), "holder_a should still hold write_a"); + assert!(!is_eof(read_b), "holder_b should still hold write_b"); + + assert!(holder_a.release(Duration::from_secs(5))); + assert!(is_eof(read_a), "releasing holder_a should close write_a"); + assert!( + !is_eof(read_b), + "releasing holder_a must not affect holder_b's write_b" + ); + + assert!(holder_b.release(Duration::from_secs(5))); + assert!(is_eof(read_b), "releasing holder_b should close write_b"); + + // SAFETY: our own read ends, still open. + unsafe { + libc::close(read_a); + libc::close(read_b); + } + } + + /// A holder must close inherited descriptors outside its assigned chunk, or + /// those descriptors can keep unrelated pipes or resources alive. + #[test] + fn holder_closes_fds_outside_its_chunk() { + let mut pipe_out = [0i32; 2]; + // SAFETY: `pipe_out` points to two valid `i32`s, as `pipe(2)` + // requires. + unsafe { + assert_eq!(libc::pipe(pipe_out.as_mut_ptr()), 0); + } + let [read_out, write_out] = pipe_out; + set_nonblocking(read_out); + + let holder = FdHolder::spawn(&[], 0..0).unwrap(); + + // SAFETY: `write_out` was just opened by us above. + unsafe { + libc::close(write_out); + } + + assert!( + wait_for_eof(read_out, Duration::from_secs(5)), + "holder kept an fd open that it was never given ownership of" + ); + + assert!(holder.release(Duration::from_secs(5))); + // SAFETY: our own read end, still open. + unsafe { + libc::close(read_out); + } + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 1911e56b1..ca29fde3a 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -1,13 +1,13 @@ +use crate::ebpf::poller::{RingBufferPoller, ThreadedRingBufferPoller}; use crate::prelude::*; use libbpf_rs::Link; use libbpf_rs::skel::OpenSkel; use libbpf_rs::skel::SkelBuilder; use std::collections::HashMap; use std::mem::MaybeUninit; +use std::os::fd::{AsFd, AsRawFd, RawFd}; use std::path::Path; -use crate::ebpf::poller::{RingBufferPoller, ThreadedRingBufferPoller}; - mod token { include!(concat!(env!("OUT_DIR"), "/memtrack_token.skel.rs")); } @@ -18,10 +18,10 @@ mod legacy { #[macro_use] mod macros; mod allocator; +mod fd_holder; mod maps; mod rmap; mod tracking; - pub use maps::OwnershipMaps; pub use rmap::RmapSupport; @@ -301,27 +301,31 @@ impl MemtrackBpf { self.probes.len() } - /// Detach all BPF links in parallel. Closing a uprobe link blocks on two - /// RCU grace periods in the kernel, but concurrent waiters share grace - /// periods, so closing from many threads scales near-linearly. + /// Detach all BPF links without blocking on kernel link teardown: forked + /// holders own disjoint fd chunks and perform the terminal close in + /// parallel, while this process only drops its duplicate references. pub fn detach_probes(&mut self) { - const DETACH_THREADS: usize = 32; - - let mut probes = std::mem::take(&mut self.probes); + let probes = std::mem::take(&mut self.probes); if probes.is_empty() { return; } debug!("Detaching {} BPF links", probes.len()); let start = std::time::Instant::now(); - let chunk_size = probes.len().div_ceil(DETACH_THREADS); - std::thread::scope(|scope| { - while !probes.is_empty() { - let split_at = probes.len().saturating_sub(chunk_size); - let chunk = probes.split_off(split_at); - scope.spawn(move || drop(chunk)); - } - }); + + let fds: Vec = probes.iter().map(|p| p.as_fd().as_raw_fd()).collect(); + let holders = fd_holder::FdHolderSet::spawn(&fds, 32); + let holder_count = holders.len(); + + drop(probes); + + if !holders.is_empty() && !holders.release_all(std::time::Duration::from_secs(30)) { + warn!( + "Link teardown is stuck in the kernel; abandoning {holder_count} fd holder processes" + ); + return; + } + debug!("Detached BPF links in {:?}", start.elapsed()); } } From 2a7a839b7aa852ab759324ed2130152a6e334a57 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 7 Sep 2026 12:04:22 +0200 Subject: [PATCH 21/21] feat(runner): add experimental flag to capture allocation stacks --- src/cli/exec/mod.rs | 1 + src/cli/experimental.rs | 12 ++++++++++++ src/cli/run/mod.rs | 2 ++ src/executor/config.rs | 8 ++++++++ src/executor/memory/executor.rs | 3 +++ 5 files changed, 26 insertions(+) diff --git a/src/cli/exec/mod.rs b/src/cli/exec/mod.rs index 4a757f74b..e7ed8b5e2 100644 --- a/src/cli/exec/mod.rs +++ b/src/cli/exec/mod.rs @@ -93,6 +93,7 @@ fn build_orchestrator_config( cycle_estimation: args.shared.cycle_estimation, exclude_allocations: args.shared.exclude_allocations, simulation_track_subprocess: args.shared.simulation_track_subprocess, + memory_capture_stack: args.shared.experimental.experimental_memory_capture_stack, }) } diff --git a/src/cli/experimental.rs b/src/cli/experimental.rs index 74aab8177..196967568 100644 --- a/src/cli/experimental.rs +++ b/src/cli/experimental.rs @@ -17,6 +17,15 @@ pub struct ExperimentalArgs { )] pub experimental_fair_sched: bool, + /// Capture allocation call stacks in memory mode. + #[arg( + long, + default_value_t = false, + help_heading = "Experimental", + env = "CODSPEED_EXPERIMENTAL_MEMORY_CAPTURE_STACK" + )] + pub experimental_memory_capture_stack: bool, + /// Deprecated: cycle estimation is enabled by default and this flag has no effect. #[arg(long, hide = true, env = "CODSPEED_EXPERIMENTAL_CYCLE_ESTIMATION")] pub experimental_cycle_estimation: bool, @@ -33,6 +42,9 @@ impl ExperimentalArgs { if self.experimental_fair_sched { flags.push("--experimental-fair-sched"); } + if self.experimental_memory_capture_stack { + flags.push("--experimental-memory-capture-stack"); + } flags } diff --git a/src/cli/run/mod.rs b/src/cli/run/mod.rs index a218155c7..86bc8440a 100644 --- a/src/cli/run/mod.rs +++ b/src/cli/run/mod.rs @@ -81,6 +81,7 @@ impl RunArgs { }, experimental: ExperimentalArgs { experimental_fair_sched: false, + experimental_memory_capture_stack: false, experimental_cycle_estimation: false, experimental_exclude_allocations: false, }, @@ -135,6 +136,7 @@ fn build_orchestrator_config( cycle_estimation: args.shared.cycle_estimation, exclude_allocations: args.shared.exclude_allocations, simulation_track_subprocess: args.shared.simulation_track_subprocess, + memory_capture_stack: args.shared.experimental.experimental_memory_capture_stack, }) } diff --git a/src/executor/config.rs b/src/executor/config.rs index 07f99c820..e39e559e3 100644 --- a/src/executor/config.rs +++ b/src/executor/config.rs @@ -98,6 +98,9 @@ pub struct OrchestratorConfig { /// Inherit valgrind's instrumentation state across a traced exec, so the cost of /// subprocesses spawned by a benchmark is measured too. pub simulation_track_subprocess: bool, + /// Capture allocation call stacks in memory mode, so allocations can be + /// attributed to the code that made them. + pub memory_capture_stack: bool, } /// Per-execution configuration passed to executors. @@ -138,6 +141,9 @@ pub struct ExecutorConfig { /// Inherit valgrind's instrumentation state across a traced exec, so the cost of /// subprocesses spawned by a benchmark is measured too. pub simulation_track_subprocess: bool, + /// Capture allocation call stacks in memory mode, so allocations can be + /// attributed to the code that made them. + pub memory_capture_stack: bool, } #[derive(Debug, Clone, PartialEq)] @@ -210,6 +216,7 @@ impl OrchestratorConfig { cycle_estimation: self.cycle_estimation, exclude_allocations: self.exclude_allocations, simulation_track_subprocess: self.simulation_track_subprocess, + memory_capture_stack: self.memory_capture_stack, } } } @@ -245,6 +252,7 @@ impl OrchestratorConfig { cycle_estimation: true, exclude_allocations: false, simulation_track_subprocess: false, + memory_capture_stack: false, } } } diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index b8a294d8a..6e53a3e9b 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -71,6 +71,9 @@ impl MemoryExecutor { cmd_builder.arg("--ipc-server"); cmd_builder.arg(server_name); cmd_builder.arg(bench_command); + if execution_context.config.memory_capture_stack { + cmd_builder.env("CODSPEED_MEMTRACK_CAPTURE_STACKS", "1"); + } // Set working directory if specified if let Some(cwd) = &execution_context.config.working_directory {