diff --git a/containers/process.go b/containers/process.go index d677fbf..e3437b0 100644 --- a/containers/process.go +++ b/containers/process.go @@ -91,6 +91,15 @@ func (p *Process) isHostNs() bool { } func (p *Process) instrument(tracer *ebpftracer.Tracer) { + if delay := *flags.InstrumentationDelay; delay > 0 && !p.StartedAt.IsZero() { + if wait := delay - time.Since(p.StartedAt); wait > 0 { + select { + case <-p.ctx.Done(): + return + case <-time.After(wait): + } + } + } b := backoff.Backoff{Factor: 2, Min: time.Second, Max: time.Minute} for { select { diff --git a/ebpftracer/tracer.go b/ebpftracer/tracer.go index 959e95b..6b4cf11 100644 --- a/ebpftracer/tracer.go +++ b/ebpftracer/tracer.go @@ -608,6 +608,16 @@ func (t *Tracer) ebpf(ch chan<- Event) error { t.collection = c t.programInstructions = programInstructions(c) + // Uprobe programs are attached per process, on demand. Register them all + // before any tracepoint or kprobe is attached: events start flowing as + // soon as the first one is, and a process handled before its uprobe + // program was registered would never get its probes. + for _, programSpec := range collectionSpec.Programs { + if strings.HasPrefix(programSpec.SectionName, "uprobe/") || strings.HasPrefix(programSpec.SectionName, "uretprobe/") { + t.uprobes[programSpec.Name] = c.Programs[programSpec.Name] + } + } + if t.enableLLMCapture { if err := c.Maps["llm_capture_config"].Update(uint32(0), uint32(1), ebpf.UpdateAny); err != nil { return fmt.Errorf("failed to enable LLM capture: %w", err) @@ -684,8 +694,7 @@ func (t *Tracer) ebpf(ch chan<- Event) error { l, err = link.Tracepoint(parts[0], parts[1], program, nil) case ebpf.Kprobe: if strings.HasPrefix(programSpec.SectionName, "uprobe/") || strings.HasPrefix(programSpec.SectionName, "uretprobe/") { - t.uprobes[programSpec.Name] = program - continue + continue // registered at load, attached to processes on demand } l, err = link.Kprobe(programSpec.AttachTo, program, nil) if err != nil && programSpec.SectionName == "kprobe/nf_ct_deliver_cached_events" { diff --git a/flags/flags.go b/flags/flags.go index ad95f9a..477ad89 100644 --- a/flags/flags.go +++ b/flags/flags.go @@ -27,6 +27,7 @@ var ( // LLM traffic, or no interest in it, should not pay for it. EnableLLMCapture = kingpin.Flag("enable-llm-capture", "Capture LLM API traffic and export token usage metrics").Default("false").Envar("ENABLE_LLM_CAPTURE").Bool() DisableGPUMonitoring = kingpin.Flag("disable-gpu-monitoring", "Disable GPU monitoring (NVML)").Default("false").Envar("DISABLE_GPU_MONITORING").Bool() + InstrumentationDelay = kingpin.Flag("instrumentation-delay", "Delay before enabling Python GIL and Node.js event loop instrumentation, after a process is started (0 disables)").Default("0s").Envar("INSTRUMENTATION_DELAY").Duration() ContainerAllowlist = kingpin.Flag("container-allowlist", "List of allowed containers (regex patterns)").Envar("CONTAINER_ALLOWLIST").Strings() ContainerDenylist = kingpin.Flag("container-denylist", "List of denied containers (regex patterns)").Envar("CONTAINER_DENYLIST").Strings()