From 61a0c0ba552dae2862e1d4c1d91d2826d3aa5207 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Thu, 8 Oct 2026 15:34:18 +0530 Subject: [PATCH 1/3] fix: register uprobe programs before attaching any tracepoint or kprobe Port of coroot/coroot-node-agent@2c72586 ("register uprobes before reporting the running processes"). Uprobe programs were registered in t.uprobes inside the attach loop, interleaved with attaching tracepoints and kprobes. Events start flowing once the first of those is attached, and handleEvents (already running) could attach TLS probes for a process before its program was registered: the attach failed, the process was marked as checked and never retried. Registering them right after the collection loads also removes concurrent writes to t.uprobes while handleEvents reads it. --- ebpftracer/tracer.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ebpftracer/tracer.go b/ebpftracer/tracer.go index 959e95ba..6b4cf11a 100644 --- a/ebpftracer/tracer.go +++ b/ebpftracer/tracer.go @@ -608,6 +608,16 @@ func (t *Tracer) ebpf(ch chan<- Event) error { t.collection = c t.programInstructions = programInstructions(c) + // Uprobe programs are attached per process, on demand. Register them all + // before any tracepoint or kprobe is attached: events start flowing as + // soon as the first one is, and a process handled before its uprobe + // program was registered would never get its probes. + for _, programSpec := range collectionSpec.Programs { + if strings.HasPrefix(programSpec.SectionName, "uprobe/") || strings.HasPrefix(programSpec.SectionName, "uretprobe/") { + t.uprobes[programSpec.Name] = c.Programs[programSpec.Name] + } + } + if t.enableLLMCapture { if err := c.Maps["llm_capture_config"].Update(uint32(0), uint32(1), ebpf.UpdateAny); err != nil { return fmt.Errorf("failed to enable LLM capture: %w", err) @@ -684,8 +694,7 @@ func (t *Tracer) ebpf(ch chan<- Event) error { l, err = link.Tracepoint(parts[0], parts[1], program, nil) case ebpf.Kprobe: if strings.HasPrefix(programSpec.SectionName, "uprobe/") || strings.HasPrefix(programSpec.SectionName, "uretprobe/") { - t.uprobes[programSpec.Name] = program - continue + continue // registered at load, attached to processes on demand } l, err = link.Kprobe(programSpec.AttachTo, program, nil) if err != nil && programSpec.SectionName == "kprobe/nf_ct_deliver_cached_events" { From 636c0ff5a1c3a01853660147b40f94c7c956841b Mon Sep 17 00:00:00 2001 From: Nikolay Sivko Date: Thu, 30 Apr 2026 11:05:23 -0300 Subject: [PATCH 2/3] add configurable startup delay for Python GIL and Node.js event loop instrumentation (cherry picked from commit 44e3e8ef582f04dabe44fa7ed78d0838c5e43aae) In this fork the delay covers Python GIL probes and the opt-in Node.js and .NET instrumentation; TLS probes are attached per connection and are not delayed. Upstream's instrumentDone channel belongs to its uprobe dedupe and is not taken. --- containers/process.go | 9 +++++++++ flags/flags.go | 1 + 2 files changed, 10 insertions(+) diff --git a/containers/process.go b/containers/process.go index d677fbf6..e3437b0b 100644 --- a/containers/process.go +++ b/containers/process.go @@ -91,6 +91,15 @@ func (p *Process) isHostNs() bool { } func (p *Process) instrument(tracer *ebpftracer.Tracer) { + if delay := *flags.InstrumentationDelay; delay > 0 && !p.StartedAt.IsZero() { + if wait := delay - time.Since(p.StartedAt); wait > 0 { + select { + case <-p.ctx.Done(): + return + case <-time.After(wait): + } + } + } b := backoff.Backoff{Factor: 2, Min: time.Second, Max: time.Minute} for { select { diff --git a/flags/flags.go b/flags/flags.go index ad95f9a9..702dbd3e 100644 --- a/flags/flags.go +++ b/flags/flags.go @@ -27,6 +27,7 @@ var ( // LLM traffic, or no interest in it, should not pay for it. EnableLLMCapture = kingpin.Flag("enable-llm-capture", "Capture LLM API traffic and export token usage metrics").Default("false").Envar("ENABLE_LLM_CAPTURE").Bool() DisableGPUMonitoring = kingpin.Flag("disable-gpu-monitoring", "Disable GPU monitoring (NVML)").Default("false").Envar("DISABLE_GPU_MONITORING").Bool() + InstrumentationDelay = kingpin.Flag("instrumentation-delay", "Delay before enabling Python GIL and Node.js event loop instrumentation, after a process is started").Default("30s").Envar("INSTRUMENTATION_DELAY").Duration() ContainerAllowlist = kingpin.Flag("container-allowlist", "List of allowed containers (regex patterns)").Envar("CONTAINER_ALLOWLIST").Strings() ContainerDenylist = kingpin.Flag("container-denylist", "List of denied containers (regex patterns)").Envar("CONTAINER_DENYLIST").Strings() From f5e665a190d56b94b1bcdbb01c3dccfb90e67aec Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Thu, 8 Oct 2026 16:18:00 +0530 Subject: [PATCH 3/3] default --instrumentation-delay to 0 Upstream delays Python GIL and Node.js event-loop instrumentation by 30s to save the attach cost on short-lived processes. Here Node.js tracing is already opt-in, and with about 300 short-lived Python processes a minute the delay saved about 2.6 millicores of agent CPU (2290 vs 2134 ms over 60s), while hiding GIL metrics for each process's first 30s. Keep the flag and leave instrumentation immediate by default. --- flags/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flags/flags.go b/flags/flags.go index 702dbd3e..477ad893 100644 --- a/flags/flags.go +++ b/flags/flags.go @@ -27,7 +27,7 @@ var ( // LLM traffic, or no interest in it, should not pay for it. EnableLLMCapture = kingpin.Flag("enable-llm-capture", "Capture LLM API traffic and export token usage metrics").Default("false").Envar("ENABLE_LLM_CAPTURE").Bool() DisableGPUMonitoring = kingpin.Flag("disable-gpu-monitoring", "Disable GPU monitoring (NVML)").Default("false").Envar("DISABLE_GPU_MONITORING").Bool() - InstrumentationDelay = kingpin.Flag("instrumentation-delay", "Delay before enabling Python GIL and Node.js event loop instrumentation, after a process is started").Default("30s").Envar("INSTRUMENTATION_DELAY").Duration() + InstrumentationDelay = kingpin.Flag("instrumentation-delay", "Delay before enabling Python GIL and Node.js event loop instrumentation, after a process is started (0 disables)").Default("0s").Envar("INSTRUMENTATION_DELAY").Duration() ContainerAllowlist = kingpin.Flag("container-allowlist", "List of allowed containers (regex patterns)").Envar("CONTAINER_ALLOWLIST").Strings() ContainerDenylist = kingpin.Flag("container-denylist", "List of denied containers (regex patterns)").Envar("CONTAINER_DENYLIST").Strings()