From ce18e430d5166158e3f75c3df32ad406c0fc2ee9 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Tue, 6 Oct 2026 14:01:28 +0530 Subject: [PATCH 1/2] perf(ebpf): wake the proc_events reader per event instead of polling every 10 ms Perf readers are created with WakeupEvents 100: the kernel wakes a reader once a CPU's buffer holds 100 events, and below that the events wait for the reader's next deadline. Process events come a few at a time, so to act on an exec before the new program connects, the proc_events deadline was cut to 10 ms. Polling 100 times a second cost ~5 millicores on an idle node (agent CPU profile: the perf reader's epoll loop went from 0.57 s to 1.0 s per 90 s). proc_events is now created with WakeupEvents 1, so an exec wakes the reader at once, and its deadline is back to the default 100 ms. --- ebpftracer/tracer.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ebpftracer/tracer.go b/ebpftracer/tracer.go index ac07deaf..c677de26 100644 --- a/ebpftracer/tracer.go +++ b/ebpftracer/tracer.go @@ -507,6 +507,10 @@ type perfMap struct { perCPUBufferSizePages int typ perfMapType readTimeout time.Duration + // wakeupEvents is how many events a CPU's buffer collects before the + // kernel wakes the reader; 0 means 100. Below that, events wait for the + // reader's next readTimeout. + wakeupEvents int } // collectionSpecForKernel returns the compiled program variant for the running @@ -610,9 +614,12 @@ func (t *Tracer) ebpf(ch chan<- Event) error { } perfMaps := []perfMap{ - // Read as often as connect events: an exec is acted on (TLS probes - // attached) before the new program makes its first connection. - {name: "proc_events", typ: perfMapTypeProcEvents, perCPUBufferSizePages: 4, readTimeout: 10 * time.Millisecond}, + // An exec must be acted on (TLS probes attached) before the new + // program makes its first connection, so the reader is woken for + // every event. Polling instead, with a 10 ms deadline to bound that + // wait, cost ~5 millicores on an idle node: 100 wakeups a second + // for events that arrive a few times a second. + {name: "proc_events", typ: perfMapTypeProcEvents, perCPUBufferSizePages: 4, wakeupEvents: 1}, {name: "tcp_listen_events", typ: perfMapTypeTCPEvents, perCPUBufferSizePages: 4}, {name: "tcp_connect_events", typ: perfMapTypeTCPEvents, perCPUBufferSizePages: 8, readTimeout: 10 * time.Millisecond}, {name: "tcp_retransmit_events", typ: perfMapTypeTCPEvents, perCPUBufferSizePages: 4}, @@ -622,7 +629,11 @@ func (t *Tracer) ebpf(ch chan<- Event) error { // Create perf buffer readers for non-L7 events pageSize := os.Getpagesize() for _, pm := range perfMaps { - r, err := perf.NewReaderWithOptions(t.collection.Maps[pm.name], pm.perCPUBufferSizePages*pageSize, perf.ReaderOptions{WakeupEvents: 100}) + wakeup := pm.wakeupEvents + if wakeup == 0 { + wakeup = 100 + } + r, err := perf.NewReaderWithOptions(t.collection.Maps[pm.name], pm.perCPUBufferSizePages*pageSize, perf.ReaderOptions{WakeupEvents: wakeup}) if err != nil { t.Close() return fmt.Errorf("failed to create ebpf reader: %w", err) From ef001b39e53064a8765684999e9433026bdeb8e6 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Tue, 6 Oct 2026 14:48:37 +0530 Subject: [PATCH 2/2] perf(ebpf): no read deadline for a perf reader woken by every event proc_events is created with WakeupEvents 1, but runEventsReader still gave it the default 100 ms deadline, so an idle node woke its reader 10 times a second for nothing. A reader the kernel wakes for every event now has no deadline; the others keep 100 ms unless set otherwise. Close still interrupts a blocked Read. --- ebpftracer/tracer.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/ebpftracer/tracer.go b/ebpftracer/tracer.go index c677de26..dfe7d21a 100644 --- a/ebpftracer/tracer.go +++ b/ebpftracer/tracer.go @@ -506,7 +506,9 @@ type perfMap struct { name string perCPUBufferSizePages int typ perfMapType - readTimeout time.Duration + // readTimeout bounds how long events below wakeupEvents wait to be + // read; 0 means 100 ms. A reader woken for every event needs none. + readTimeout time.Duration // wakeupEvents is how many events a CPU's buffer collects before the // kernel wakes the reader; 0 means 100. Below that, events wait for the // reader's next readTimeout. @@ -616,9 +618,9 @@ func (t *Tracer) ebpf(ch chan<- Event) error { perfMaps := []perfMap{ // An exec must be acted on (TLS probes attached) before the new // program makes its first connection, so the reader is woken for - // every event. Polling instead, with a 10 ms deadline to bound that - // wait, cost ~5 millicores on an idle node: 100 wakeups a second - // for events that arrive a few times a second. + // every event and needs no deadline. Polling instead, with a 10 ms + // deadline to bound that wait, cost ~5 millicores on an idle node: + // 100 wakeups a second for events that arrive a few times a second. {name: "proc_events", typ: perfMapTypeProcEvents, perCPUBufferSizePages: 4, wakeupEvents: 1}, {name: "tcp_listen_events", typ: perfMapTypeTCPEvents, perCPUBufferSizePages: 4}, {name: "tcp_connect_events", typ: perfMapTypeTCPEvents, perCPUBufferSizePages: 8, readTimeout: 10 * time.Millisecond}, @@ -639,7 +641,11 @@ func (t *Tracer) ebpf(ch chan<- Event) error { return fmt.Errorf("failed to create ebpf reader: %w", err) } t.readers[pm.name] = r - go runEventsReader(pm.name, r, ch, pm.typ, pm.readTimeout) + readTimeout := pm.readTimeout + if readTimeout == 0 && wakeup > 1 { + readTimeout = 100 * time.Millisecond + } + go runEventsReader(pm.name, r, ch, pm.typ, readTimeout) } // Create ring buffer reader for l7_events (provides global ordering for SSE streaming) @@ -861,13 +867,14 @@ func (t *lostSamplesTracker) recordLostSamples(name string, count uint64, cpu in } } +// runEventsReader reads r until it is closed. readTimeout 0 means no deadline: +// the reader sleeps until the kernel wakes it (Close interrupts that too). func runEventsReader(name string, r *perf.Reader, ch chan<- Event, typ perfMapType, readTimeout time.Duration) { tracker := getLostSamplesTracker(name) - if readTimeout == 0 { - readTimeout = 100 * time.Millisecond - } for { - r.SetDeadline(time.Now().Add(readTimeout)) + if readTimeout > 0 { + r.SetDeadline(time.Now().Add(readTimeout)) + } rec, err := r.Read() if err != nil { if errors.Is(err, perf.ErrClosed) {