fix: register uprobe programs before attaching, add opt-in --instrumentation-delay (B4a of #369)

containers/process.go

@@ -91,6 +91,15 @@ func (p *Process) isHostNs() bool {
 }
 
 func (p *Process) instrument(tracer *ebpftracer.Tracer) {
+	if delay := *flags.InstrumentationDelay; delay > 0 && !p.StartedAt.IsZero() {
+		if wait := delay - time.Since(p.StartedAt); wait > 0 {
+			select {
+			case <-p.ctx.Done():
+				return
+			case <-time.After(wait):
+			}
+		}
+	}
 	b := backoff.Backoff{Factor: 2, Min: time.Second, Max: time.Minute}
 	for {
 		select {

ebpftracer/tracer.go

@@ -608,6 +608,16 @@ func (t *Tracer) ebpf(ch chan<- Event) error {
 	t.collection = c
 	t.programInstructions = programInstructions(c)
 
+	// Uprobe programs are attached per process, on demand. Register them all
+	// before any tracepoint or kprobe is attached: events start flowing as
+	// soon as the first one is, and a process handled before its uprobe
+	// program was registered would never get its probes.
+	for _, programSpec := range collectionSpec.Programs {
+		if strings.HasPrefix(programSpec.SectionName, "uprobe/") || strings.HasPrefix(programSpec.SectionName, "uretprobe/") {
+			t.uprobes[programSpec.Name] = c.Programs[programSpec.Name]
+		}
+	}
+
 	if t.enableLLMCapture {
 		if err := c.Maps["llm_capture_config"].Update(uint32(0), uint32(1), ebpf.UpdateAny); err != nil {
 			return fmt.Errorf("failed to enable LLM capture: %w", err)
@@ -684,8 +694,7 @@ func (t *Tracer) ebpf(ch chan<- Event) error {
 			l, err = link.Tracepoint(parts[0], parts[1], program, nil)
 		case ebpf.Kprobe:
 			if strings.HasPrefix(programSpec.SectionName, "uprobe/") || strings.HasPrefix(programSpec.SectionName, "uretprobe/") {
-				t.uprobes[programSpec.Name] = program
-				continue
+				continue // registered at load, attached to processes on demand
 			}
 			l, err = link.Kprobe(programSpec.AttachTo, program, nil)
 			if err != nil && programSpec.SectionName == "kprobe/nf_ct_deliver_cached_events" {

flags/flags.go

@@ -27,6 +27,7 @@ var (
 	// LLM traffic, or no interest in it, should not pay for it.
 	EnableLLMCapture     = kingpin.Flag("enable-llm-capture", "Capture LLM API traffic and export token usage metrics").Default("false").Envar("ENABLE_LLM_CAPTURE").Bool()
 	DisableGPUMonitoring = kingpin.Flag("disable-gpu-monitoring", "Disable GPU monitoring (NVML)").Default("false").Envar("DISABLE_GPU_MONITORING").Bool()
+	InstrumentationDelay = kingpin.Flag("instrumentation-delay", "Delay before enabling Python GIL and Node.js event loop instrumentation, after a process is started (0 disables)").Default("0s").Envar("INSTRUMENTATION_DELAY").Duration()
 
 	ContainerAllowlist = kingpin.Flag("container-allowlist", "List of allowed containers (regex patterns)").Envar("CONTAINER_ALLOWLIST").Strings()
 	ContainerDenylist  = kingpin.Flag("container-denylist", "List of denied containers (regex patterns)").Envar("CONTAINER_DENYLIST").Strings()