From ae18fd8a1f1e0e0c1d0fde02899b85cce46ff6a2 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Tue, 6 Oct 2026 15:33:15 +0530 Subject: [PATCH 1/2] fix(l7): track up to 1000 HTTP/2 streams per connection gRPC clients multiplex long-polls over one connection by the hundred. A long-poll gets no response headers until it returns, up to a minute later. At the 100-stream limit those live polls were evicted, oldest waiting first, and their results lost. On a test cluster one node with long-polling workers evicted ~350 streams a minute. Locally, 300 concurrent 5-second requests on one connection completed 800-1,110 of 3,600. The limit is now 1000. Streams whose response was lost are still reclaimed by the stream GC, and a full table is ~200 KB. --- ebpftracer/l7/http2.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ebpftracer/l7/http2.go b/ebpftracer/l7/http2.go index 79270a4..07bc2ce 100644 --- a/ebpftracer/l7/http2.go +++ b/ebpftracer/l7/http2.go @@ -107,7 +107,13 @@ const ( // Max concurrent HTTP/2 streams tracked per connection. // Prevents unbounded memory growth when responses never complete (orphan // streams); at the limit the oldest stream makes way (evictOldestRequest). - maxActiveRequests = 100 + // It must exceed what one connection legitimately carries: gRPC clients + // multiplex long-polls (a Temporal worker's task-queue polls, which get + // no response headers until the poll returns, up to a minute later) by + // the hundred over one connection, and at 100 live polls were evicted. + // Orphans are still reclaimed by the stream GC (http2DecoderGcInterval); + // a request costs ~200 bytes, so a full table is ~200 KB. + maxActiveRequests = 1000 ) type Http2FrameHeader struct { From 65a8b5917dc52dc11d1634457d5986de0b03d1a1 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Tue, 6 Oct 2026 15:33:49 +0530 Subject: [PATCH 2/2] docs(l7): describe the long-poll case without naming a product --- ebpftracer/l7/http2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebpftracer/l7/http2.go b/ebpftracer/l7/http2.go index 07bc2ce..d9be66b 100644 --- a/ebpftracer/l7/http2.go +++ b/ebpftracer/l7/http2.go @@ -108,7 +108,7 @@ const ( // Prevents unbounded memory growth when responses never complete (orphan // streams); at the limit the oldest stream makes way (evictOldestRequest). // It must exceed what one connection legitimately carries: gRPC clients - // multiplex long-polls (a Temporal worker's task-queue polls, which get + // multiplex long-polls (workflow and job-queue workers' polls, which get // no response headers until the poll returns, up to a minute later) by // the hundred over one connection, and at 100 live polls were evicted. // Orphans are still reclaimed by the stream GC (http2DecoderGcInterval);