From e548f9ccc6737ce39bf0dafa15f36b57fa619e61 Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 13:27:32 +0100 Subject: [PATCH 01/21] fix(core): initializer, gzip pool and shared-state robustness - util/initializer: bound the auto-retry wait, reap tasks that overrun their timeout, and make an attempt's start and finish atomic so a reaped attempt can never overwrite the next one's state. Wait never reports a failed task as succeeded, and an attempt that finds the app context cancelled still releases the WaitGroup ExecuteTasks holds tasksMu for. - util/gzip_pool: closing a pooled reader also closes its source body. - data: CounterInt64 pushes to remote only when the local value advanced and never overwrites a higher remote value; the keep-latest pubsub buffer keeps the fresher of two values. A refresh the caller's deadline allows past fallbackTimeout is still applied when it lands. - common: UniqueUpstreamKey no longer includes the network id, which is assigned after the upstream is created and so changed the key over the upstream's lifetime. Co-authored-by: shpookas Co-Authored-By: Claude Opus 5.5 --- clients/http_json_rpc_client.go | 20 ++- common/upstream.go | 14 +- common/upstream_test.go | 82 ++++++++++ data/redis_pubsub_manager.go | 83 +++++----- data/redis_pubsub_manager_test.go | 146 +++++++++++++++++ data/shared_state_variable.go | 37 +++-- data/shared_state_variable_test.go | 173 ++++++++++++++++++++ util/gzip_pool.go | 22 ++- util/gzip_pool_close_test.go | 130 +++++++++++++++ util/initializer.go | 237 +++++++++++++++++++-------- util/initializer_test.go | 250 ++++++++++++++++++++++++++++- util/testing.go | 2 + 12 files changed, 1062 insertions(+), 134 deletions(-) create mode 100644 common/upstream_test.go create mode 100644 data/redis_pubsub_manager_test.go create mode 100644 util/gzip_pool_close_test.go diff --git a/clients/http_json_rpc_client.go b/clients/http_json_rpc_client.go index 7145ffb0f..736668956 100644 --- a/clients/http_json_rpc_client.go +++ b/clients/http_json_rpc_client.go @@ -761,7 +761,8 @@ func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *c _ = resp.Body.Close() // Must close on error path return nil, common.NewErrEndpointTransportFailure(c.Url, fmt.Errorf("cannot create gzip reader: %w", err)) } - bodyReader = c.gzipPool.WrapGzipReader(gzReader) + // The wrapper closes resp.Body too; gzip.Reader.Close does not. + bodyReader = c.gzipPool.WrapGzipReader(gzReader, resp.Body) } nr := common.NewNormalizedResponse(). @@ -887,6 +888,13 @@ func (c *GenericHttpJsonRpcClient) normalizeJsonRpcError(r *http.Response, nr *c } } + return classifyJsonRpcError(r, nr, jr, err, c.errorExtractor, c.upstream) +} + +// classifyJsonRpcError turns a parsed upstream response into a normalized +// error (nil on success). Shared by the HTTP and WebSocket clients; WS +// callers pass a synthetic 200 response since the frame has no HTTP status. +func classifyJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *common.JsonRpcResponse, err error, extractor common.JsonRpcErrorExtractor, ups common.Upstream) error { if err != nil { e := common.NewErrJsonRpcExceptionInternal( 0, @@ -894,7 +902,7 @@ func (c *GenericHttpJsonRpcClient) normalizeJsonRpcError(r *http.Response, nr *c "could not parse json rpc response from upstream", err, map[string]interface{}{ - "upstreamId": c.upstream.Id(), + "upstreamId": ups.Id(), "statusCode": r.StatusCode, "headers": r.Header, }, @@ -921,8 +929,10 @@ func (c *GenericHttpJsonRpcClient) normalizeJsonRpcError(r *http.Response, nr *c } } - if e := c.errorExtractor.Extract(r, nr, jr, c.upstream); e != nil { - return e + if extractor != nil { + if e := extractor.Extract(r, nr, jr, ups); e != nil { + return e + } } if jr == nil || jr.Error == nil { @@ -935,7 +945,7 @@ func (c *GenericHttpJsonRpcClient) normalizeJsonRpcError(r *http.Response, nr *c "unknown json-rpc error", jr.Error, map[string]interface{}{ - "upstreamId": c.upstream.Id(), + "upstreamId": ups.Id(), "statusCode": r.StatusCode, "headers": r.Header, }, diff --git a/common/upstream.go b/common/upstream.go index 0d86efa41..6ac8ddae5 100644 --- a/common/upstream.go +++ b/common/upstream.go @@ -4,6 +4,8 @@ import ( "context" "crypto/sha256" "encoding/hex" + "maps" + "slices" "github.com/rs/zerolog" ) @@ -60,19 +62,19 @@ type Upstream interface { } // UniqueUpstreamKey returns a unique hash for an upstream. -// It is used to identify the upstream uniquely in shared-state storage. -// Sometimes ID might not be enough for example if user changes the endpoint to a completely different network. +// It is used to identify the upstream uniquely in shared-state storage and the +// client cache, so it is derived from static config only (the network id is +// unknown until registration) with headers in sorted order. func UniqueUpstreamKey(up Upstream) string { sha := sha256.New() cfg := up.Config() sha.Write([]byte(cfg.Id)) sha.Write([]byte(cfg.Endpoint)) - sha.Write([]byte(up.NetworkId())) - if cfg.JsonRpc != nil && cfg.JsonRpc.Headers != nil { - for k, v := range cfg.JsonRpc.Headers { + if cfg.JsonRpc != nil { + for _, k := range slices.Sorted(maps.Keys(cfg.JsonRpc.Headers)) { sha.Write([]byte(k)) - sha.Write([]byte(v)) + sha.Write([]byte(cfg.JsonRpc.Headers[k])) } } diff --git a/common/upstream_test.go b/common/upstream_test.go new file mode 100644 index 000000000..a4a806265 --- /dev/null +++ b/common/upstream_test.go @@ -0,0 +1,82 @@ +package common + +import ( + "testing" +) + +// stubUpstreamForKey implements only what UniqueUpstreamKey may call. +type stubUpstreamForKey struct { + cfg *UpstreamConfig + networkId string + Upstream +} + +func (s *stubUpstreamForKey) Config() *UpstreamConfig { return s.cfg } +func (s *stubUpstreamForKey) NetworkId() string { return s.networkId } + +func TestUniqueUpstreamKey_StableAcrossNetworkIdChanges(t *testing.T) { + cfg := &UpstreamConfig{Id: "u1", Endpoint: "https://example/0"} + + // NetworkId reads "n/a" until registration; the key must not change. + pre := UniqueUpstreamKey(&stubUpstreamForKey{cfg: cfg, networkId: "n/a"}) + post := UniqueUpstreamKey(&stubUpstreamForKey{cfg: cfg, networkId: "evm:1"}) + + if pre != post { + t.Fatalf("UniqueUpstreamKey changed when NetworkId changed: pre=%q post=%q", pre, post) + } +} + +func TestUniqueUpstreamKey_DeterministicAcrossHeaderOrder(t *testing.T) { + mk := func(headers map[string]string) string { + cfg := &UpstreamConfig{ + Id: "u1", + Endpoint: "https://example/0", + JsonRpc: &JsonRpcUpstreamConfig{Headers: headers}, + } + return UniqueUpstreamKey(&stubUpstreamForKey{cfg: cfg, networkId: "evm:1"}) + } + + // Map iteration order is randomised; the key must not depend on it. + headers := map[string]string{ + "Authorization": "Bearer xyz", + "X-Tenant": "tenant-a", + "X-Region": "eu-west-1", + "X-Custom": "value", + } + first := mk(headers) + for i := range 50 { + got := mk(headers) + if got != first { + t.Fatalf("UniqueUpstreamKey non-deterministic across map iterations: first=%q got=%q (iter %d)", first, got, i) + } + } +} + +func TestUniqueUpstreamKey_DistinctConfigsProduceDistinctKeys(t *testing.T) { + base := &UpstreamConfig{Id: "u1", Endpoint: "https://example/0"} + keyBase := UniqueUpstreamKey(&stubUpstreamForKey{cfg: base, networkId: "evm:1"}) + + cases := []struct { + name string + cfg *UpstreamConfig + }{ + {"different id", &UpstreamConfig{Id: "u2", Endpoint: "https://example/0"}}, + {"different endpoint", &UpstreamConfig{Id: "u1", Endpoint: "https://example/1"}}, + {"add headers", &UpstreamConfig{ + Id: "u1", Endpoint: "https://example/0", + JsonRpc: &JsonRpcUpstreamConfig{Headers: map[string]string{"X": "1"}}, + }}, + {"different header value", &UpstreamConfig{ + Id: "u1", Endpoint: "https://example/0", + JsonRpc: &JsonRpcUpstreamConfig{Headers: map[string]string{"X": "2"}}, + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := UniqueUpstreamKey(&stubUpstreamForKey{cfg: tc.cfg, networkId: "evm:1"}) + if got == keyBase { + t.Fatalf("expected distinct key, got same as base: %q", got) + } + }) + } +} diff --git a/data/redis_pubsub_manager.go b/data/redis_pubsub_manager.go index 57e8da3ad..a86ed2e9f 100644 --- a/data/redis_pubsub_manager.go +++ b/data/redis_pubsub_manager.go @@ -15,11 +15,43 @@ import ( "github.com/rs/zerolog" ) -// subscriberChannel wraps a channel with metadata to prevent double-close +// subscriberChannel wraps the delivery channel with a done signal, closed +// once, so senders never need a lock or risk sending on a closed channel. type subscriberChannel struct { - ch chan CounterInt64State - closed bool - mu sync.Mutex + ch chan CounterInt64State + done chan struct{} + closeOnce sync.Once +} + +// close marks the subscriber as gone. It is idempotent and leaves ch open. +func (sc *subscriberChannel) close() { + sc.closeOnce.Do(func() { close(sc.done) }) +} + +// sendKeepLatest delivers value without blocking: when the buffer is full, +// the fresher of the buffered and new value (by UpdatedAt) is kept, so a +// monotonic counter never loses its newest value to a slow consumer. +func (sc *subscriberChannel) sendKeepLatest(value CounterInt64State) { + for { + select { + case sc.ch <- value: + return + case <-sc.done: + return + default: + } + // Buffer full: drain it and retry with whichever value is fresher. + select { + case queued := <-sc.ch: + if queued.UpdatedAt > value.UpdatedAt { + value = queued + } + case <-sc.done: + return + default: + // A consumer drained it concurrently; retry. + } + } } // RedisPubSubManager is a self-healing manager for Redis pubsub subscriptions. @@ -108,14 +140,8 @@ func (m *RedisPubSubManager) stop() { // Close all subscriber channels m.subscribers.Range(func(key, value interface{}) bool { - channels := value.([]*subscriberChannel) - for _, sc := range channels { - sc.mu.Lock() - if !sc.closed { - close(sc.ch) - sc.closed = true - } - sc.mu.Unlock() + for _, sc := range value.([]*subscriberChannel) { + sc.close() } return true }) @@ -191,7 +217,8 @@ func (m *RedisPubSubManager) Subscribe(key string) (<-chan CounterInt64State, fu } sc := &subscriberChannel{ - ch: make(chan CounterInt64State, 1), + ch: make(chan CounterInt64State, 1), + done: make(chan struct{}), } // Add the channel to subscribers @@ -200,26 +227,14 @@ func (m *RedisPubSubManager) Subscribe(key string) (<-chan CounterInt64State, fu // Get initial value in background go func() { if val, ok, err := m.getCurrentValue(m.appCtx, key); err == nil && ok { - sc.mu.Lock() - if !sc.closed { - select { - case sc.ch <- val: - case <-m.appCtx.Done(): - } - } - sc.mu.Unlock() + sc.sendKeepLatest(val) } }() // Return cleanup function cleanup := func() { m.removeSubscriber(key, sc) - sc.mu.Lock() - if !sc.closed { - close(sc.ch) - sc.closed = true - } - sc.mu.Unlock() + sc.close() } return sc.ch, cleanup, nil @@ -444,17 +459,7 @@ func (m *RedisPubSubManager) notifySubscribers(key string, value CounterInt64Sta if !ok { return } - - channels := subsValue.([]*subscriberChannel) - for _, sc := range channels { - sc.mu.Lock() - if !sc.closed { - select { - case sc.ch <- value: - default: - // Channel is full, skip - } - } - sc.mu.Unlock() + for _, sc := range subsValue.([]*subscriberChannel) { + sc.sendKeepLatest(value) } } diff --git a/data/redis_pubsub_manager_test.go b/data/redis_pubsub_manager_test.go new file mode 100644 index 000000000..6f7c25248 --- /dev/null +++ b/data/redis_pubsub_manager_test.go @@ -0,0 +1,146 @@ +package data + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func newTestSubscriberChannel() *subscriberChannel { + return &subscriberChannel{ + ch: make(chan CounterInt64State, 1), + done: make(chan struct{}), + } +} + +// Updates arriving faster than the consumer drains its buffer must evict the +// stale buffered value rather than drop the newer one. +func TestSubscriberChannel_SendKeepLatestEvictsStale(t *testing.T) { + sc := newTestSubscriberChannel() + + first := CounterInt64State{Value: 100, UpdatedAt: 1} + second := CounterInt64State{Value: 101, UpdatedAt: 2} + third := CounterInt64State{Value: 102, UpdatedAt: 3} + + // No consumer — three publishes land back-to-back, overflowing the + // single-slot buffer twice. Only the latest value must survive. + sc.sendKeepLatest(first) + sc.sendKeepLatest(second) + sc.sendKeepLatest(third) + + select { + case got := <-sc.ch: + assert.Equal(t, third, got, "full buffer must be updated to the latest published value") + default: + t.Fatalf("subscriber channel empty after three publishes") + } + + select { + case extra := <-sc.ch: + t.Fatalf("unexpected extra value on subscriber channel: %+v", extra) + default: + } +} + +// A stale value (e.g. Subscribe's initial GET landing after a pubsub +// message) must not replace a fresher one still in the buffer. +func TestSubscriberChannel_SendKeepLatestKeepsFresherBuffered(t *testing.T) { + sc := newTestSubscriberChannel() + + fresh := CounterInt64State{Value: 101, UpdatedAt: 2} + sc.sendKeepLatest(fresh) + sc.sendKeepLatest(CounterInt64State{Value: 100, UpdatedAt: 1}) + + assert.Equal(t, fresh, <-sc.ch, "buffered fresher value must survive a stale send") +} + +func TestSubscriberChannel_SendKeepLatestSequentialDeliversEachValue(t *testing.T) { + sc := newTestSubscriberChannel() + + for _, v := range []int64{10, 11, 12, 13} { + st := CounterInt64State{Value: v, UpdatedAt: v} + sc.sendKeepLatest(st) + got := <-sc.ch + assert.Equal(t, st, got, "sequential publish+drain should deliver each value untouched") + } +} + +// Sends to a closed subscriber return immediately instead of blocking. +func TestSubscriberChannel_SendKeepLatestReturnsAfterClose(t *testing.T) { + sc := newTestSubscriberChannel() + + // Fill the buffer so a subsequent send would spin in the drain loop + // if it didn't respect the done signal. + sc.ch <- CounterInt64State{Value: 1, UpdatedAt: 1} + sc.close() + + done := make(chan struct{}) + go func() { + defer close(done) + sc.sendKeepLatest(CounterInt64State{Value: 99, UpdatedAt: 99}) + }() + + select { + case <-done: + // ok — send returned promptly + case <-time.After(time.Second): + t.Fatalf("sendKeepLatest did not return after subscriber was closed") + } +} + +// stop() and the per-subscriber cleanup can both close a subscriber. +func TestSubscriberChannel_CloseIsIdempotent(t *testing.T) { + sc := newTestSubscriberChannel() + sc.close() + sc.close() + sc.close() + + select { + case <-sc.done: + // expected + default: + t.Fatalf("done channel must be closed after close()") + } +} + +// notifySubscribers keeps the latest value on every channel for the key. +func TestRedisPubSubManager_NotifySubscribersKeepsLatest(t *testing.T) { + m := &RedisPubSubManager{} + a := newTestSubscriberChannel() + b := newTestSubscriberChannel() + m.addSubscriber("finalized", a) + m.addSubscriber("finalized", b) + + m.notifySubscribers("finalized", CounterInt64State{Value: 1, UpdatedAt: 1}) + m.notifySubscribers("finalized", CounterInt64State{Value: 2, UpdatedAt: 2}) + + for _, sc := range []*subscriberChannel{a, b} { + got := <-sc.ch + assert.Equal(t, int64(2), got.Value, "each subscriber should observe the latest value") + } +} + +// Concurrent sends and closes never panic or block. +func TestSubscriberChannel_ConcurrentSendAndClose(t *testing.T) { + for trial := 0; trial < 20; trial++ { + sc := newTestSubscriberChannel() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + sc.sendKeepLatest(CounterInt64State{Value: int64(i), UpdatedAt: int64(i + 1)}) + } + }() + go func() { + defer wg.Done() + // Close mid-burst. + time.Sleep(time.Microsecond * 50) + sc.close() + }() + wg.Wait() + } +} diff --git a/data/shared_state_variable.go b/data/shared_state_variable.go index 80b891830..1c4a96726 100644 --- a/data/shared_state_variable.go +++ b/data/shared_state_variable.go @@ -460,17 +460,27 @@ func (c *counterInt64) TryUpdateIfStale(ctx context.Context, staleness time.Dura // (e.g. Redis lock acquisition) can block normal request flow. span.SetAttributes(attribute.Bool("foreground_remote_io_disabled", true)) - // Execute the refresh function (e.g., RPC call to get latest block) in background + // Execute the refresh function (e.g., RPC call to get latest block) in background. + // fallbackTimeout is sized for remote storage ops; honor a longer caller + // deadline since the refresh itself can legitimately take longer. The + // parent stays appCtx so the fetch survives foreground cancellation. + fnTimeout := c.registry.fallbackTimeout + if deadline, ok := ctx.Deadline(); ok { + if remaining := time.Until(deadline); remaining > fnTimeout { + fnTimeout = remaining + } + } + resultCh := make(chan refreshResult, 1) go func() { - fnCtx, fnCancel := context.WithTimeout(c.registry.appCtx, c.registry.fallbackTimeout) + fnCtx, fnCancel := context.WithTimeout(c.registry.appCtx, fnTimeout) defer fnCancel() // Create a span for the actual RPC/refresh call - this is usually what takes time _, fnSpan := common.StartSpan(ctx, "CounterInt64.TryUpdateIfStale.ExecuteRefresh", trace.WithAttributes( attribute.String("key", c.key), - attribute.Int64("timeout_ms", c.registry.fallbackTimeout.Milliseconds()), + attribute.Int64("timeout_ms", fnTimeout.Milliseconds()), ), ) value, err := executeNewValueFn(fnCtx) @@ -503,7 +513,7 @@ func (c *counterInt64) TryUpdateIfStale(ctx context.Context, staleness time.Dura attribute.Bool("async_refresh", true), ) c.markFreshAttempt() - c.continueAsyncRefresh(resultCh) + c.continueAsyncRefresh(resultCh, fnTimeout) return initialVal, nil } } @@ -541,14 +551,14 @@ func (c *counterInt64) applyRefreshResult(initialVal int64, r refreshResult) (in return c.value.Load(), nil } -func (c *counterInt64) continueAsyncRefresh(resultCh <-chan refreshResult) { +func (c *counterInt64) continueAsyncRefresh(resultCh <-chan refreshResult, fnTimeout time.Duration) { go func() { // Bound the helper's lifetime. The paired refresh worker is capped by - // fnCtx (fallbackTimeout), but if a downstream call ignores its context - // the worker may never send and this goroutine would leak permanently. - // Cap at fallbackTimeout + a small buffer; drop the eventual value in - // the pathological case — the next TryUpdateIfStale will refresh again. - timer := time.NewTimer(c.registry.fallbackTimeout + time.Second) + // fnCtx (fnTimeout), but if a downstream call ignores its context the + // worker may never send and this goroutine would leak permanently. + // Cap at fnTimeout + a small buffer; drop the eventual value in the + // pathological case — the next TryUpdateIfStale will refresh again. + timer := time.NewTimer(fnTimeout + time.Second) defer timer.Stop() select { @@ -730,6 +740,13 @@ func (c *counterInt64) scheduleBackgroundPushCurrent() { local = c.localState() } + // Remote ahead by no more than ignoreRollbackOf means we are + // lagging: adopt it instead of overwriting it with a lower value. + if remoteOk && remote.Value > local.Value && remote.Value-local.Value <= c.ignoreRollbackOf { + c.processNewValue(UpdateSourceRemoteCheck, remote.Value) + return + } + // If our local state is newer (or remote missing), push it to remote. // Skip only if UpdatedAt indicates uninitialized; Value can be 0 (e.g., earliest = genesis) if local.UpdatedAt <= 0 { diff --git a/data/shared_state_variable_test.go b/data/shared_state_variable_test.go index 026abdcc2..52a25f2ed 100644 --- a/data/shared_state_variable_test.go +++ b/data/shared_state_variable_test.go @@ -841,6 +841,96 @@ func TestCounterInt64_TryUpdateIfStale_NoThunderingHerdOnError(t *testing.T) { connector.AssertExpectations(t) } +// The background refresh gets at least the caller's context deadline rather +// than being capped at fallbackTimeout. +func TestCounterInt64_TryUpdateIfStale_FnTimeoutFromCtxDeadline(t *testing.T) { + // refreshFn deliberately returns an error so applyRefreshResult short-circuits + // before scheduleBackgroundPushCurrent, avoiding a background Publish goroutine + // that would race across t.Run boundaries. + refreshErr := errors.New("intentional") + makeCounter := func(t *testing.T, registry *sharedStateRegistry, key string) *counterInt64 { + t.Helper() + c := &counterInt64{ + registry: registry, + key: key, + ignoreRollbackOf: 1024, + } + c.value.Store(5) + c.updatedAtUnixMs.Store(time.Now().Add(-2 * time.Second).UnixMilli()) + return c + } + + t.Run("ctx deadline longer than fallback is honored", func(t *testing.T) { + registry, _, _ := setupTest("my-dev") + registry.fallbackTimeout = 500 * time.Millisecond + registry.updateMaxWait = 5 * time.Second + + counter := makeCounter(t, registry, "test-ctx-deadline") + + const callerDeadline = 4 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), callerDeadline) + defer cancel() + + var fnTimeout time.Duration + refreshFn := func(fnCtx context.Context) (int64, error) { + if deadline, ok := fnCtx.Deadline(); ok { + fnTimeout = time.Until(deadline) + } + return 0, refreshErr + } + + _, err := counter.TryUpdateIfStale(ctx, time.Second, refreshFn) + assert.ErrorIs(t, err, refreshErr) + assert.Greater(t, fnTimeout, registry.fallbackTimeout, + "fn timeout should honor caller ctx deadline when longer than fallbackTimeout") + assert.LessOrEqual(t, fnTimeout, callerDeadline) + }) + + t.Run("no ctx deadline falls back to fallbackTimeout", func(t *testing.T) { + registry, _, _ := setupTest("my-dev") + registry.fallbackTimeout = 500 * time.Millisecond + + counter := makeCounter(t, registry, "test-no-deadline") + + var fnTimeout time.Duration + refreshFn := func(fnCtx context.Context) (int64, error) { + if deadline, ok := fnCtx.Deadline(); ok { + fnTimeout = time.Until(deadline) + } + return 0, refreshErr + } + + _, err := counter.TryUpdateIfStale(context.Background(), time.Second, refreshFn) + assert.ErrorIs(t, err, refreshErr) + assert.LessOrEqual(t, fnTimeout, registry.fallbackTimeout, + "fn timeout should be bounded by fallbackTimeout when caller has no deadline") + }) + + t.Run("ctx deadline shorter than fallback uses fallback", func(t *testing.T) { + registry, _, _ := setupTest("my-dev") + registry.fallbackTimeout = 2 * time.Second + registry.updateMaxWait = 5 * time.Second + + counter := makeCounter(t, registry, "test-short-deadline") + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + var fnTimeout time.Duration + refreshFn := func(fnCtx context.Context) (int64, error) { + if deadline, ok := fnCtx.Deadline(); ok { + fnTimeout = time.Until(deadline) + } + return 0, refreshErr + } + + _, err := counter.TryUpdateIfStale(ctx, time.Second, refreshFn) + assert.ErrorIs(t, err, refreshErr) + assert.Greater(t, fnTimeout, 500*time.Millisecond, + "fn timeout should use fallbackTimeout when caller deadline is shorter") + }) +} + func TestCounterInt64_ReaderStarvation(t *testing.T) { t.Run("GetValue NOT blocked by long-running TryUpdateIfStale", func(t *testing.T) { counter := &counterInt64{ @@ -1743,6 +1833,53 @@ func TestCounterInt64_FresherLocalPushesToStaleRemote(t *testing.T) { }) } +func TestCounterInt64_TryUpdate_NoRemoteWriteWithoutAdvance(t *testing.T) { + registry, connector, ctx := setupTest("no-advance") + connector.On("PublishCounterInt64", mock.Anything, "test", mock.Anything).Return(nil).Maybe() + connector.On("Set", mock.Anything, "test", "value", mock.Anything, mock.Anything).Return(nil).Maybe() + + counter := &counterInt64{ + registry: registry, + key: "test", + ignoreRollbackOf: 1024, + } + counter.value.Store(100) + counter.updatedAtUnixMs.Store(time.Now().UnixMilli()) + + assert.Equal(t, int64(100), counter.TryUpdate(ctx, 100)) + assert.Equal(t, int64(100), counter.TryUpdate(ctx, 99)) + time.Sleep(50 * time.Millisecond) + connector.AssertNotCalled(t, "Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + connector.AssertNotCalled(t, "PublishCounterInt64", mock.Anything, mock.Anything, mock.Anything) +} + +// A lagging instance advancing its local value must not overwrite a higher +// remote value just because its local timestamp is newer. +func TestCounterInt64_BackgroundPush_DoesNotRegressHigherRemote(t *testing.T) { + registry, connector, ctx := setupTest("no-regress") + + lock := &MockLock{} + lock.On("Unlock", mock.Anything).Return(nil) + connector.On("Lock", mock.Anything, "test", mock.Anything).Return(lock, nil) + connector.On("PublishCounterInt64", mock.Anything, "test", mock.Anything).Return(nil).Maybe() + connector.On("Get", mock.Anything, ConnectorMainIndex, "test", "value", nil). + Return([]byte(`{"v":105,"t":1,"b":"other-instance"}`), nil) + connector.On("Set", mock.Anything, "test", "value", mock.Anything, mock.Anything).Return(nil).Maybe() + + counter := &counterInt64{ + registry: registry, + key: "test", + ignoreRollbackOf: 1024, + } + counter.value.Store(100) + counter.updatedAtUnixMs.Store(time.Now().UnixMilli()) + + counter.TryUpdate(ctx, 101) + assert.Eventually(t, func() bool { return counter.GetValue() == 105 }, time.Second, 10*time.Millisecond, + "local counter should adopt the higher remote value") + connector.AssertNotCalled(t, "Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + // TestCounterInt64_ZeroIsNotAHeadObservation covers the case where a shared head // counter is asked to move to zero. // @@ -1862,3 +1999,39 @@ func TestCounterInt64_ZeroIsNotAHeadObservation(t *testing.T) { assert.Equal(t, int64(0), counter.GetValue()) }) } + +// A refresh that outlives updateMaxWait finishes in the background. When the +// caller's deadline extends fnTimeout past fallbackTimeout, a result that +// arrives after fallbackTimeout must still be applied. +func TestCounterInt64_TryUpdateIfStale_AsyncResultWithinExtendedTimeoutIsApplied(t *testing.T) { + registry, connector, _ := setupTest("my-dev") + registry.fallbackTimeout = 200 * time.Millisecond + registry.updateMaxWait = 20 * time.Millisecond + connector.On("PublishCounterInt64", mock.Anything, "my-dev/test-async-extended", mock.Anything).Return(nil).Maybe() + connector.On("Lock", mock.Anything, "my-dev/test-async-extended", mock.Anything).Return(nil, errors.New("not acquired")).Maybe() + + counter := &counterInt64{registry: registry, key: "my-dev/test-async-extended", ignoreRollbackOf: 1024} + counter.value.Store(5) + counter.updatedAtUnixMs.Store(time.Now().Add(-2 * time.Second).UnixMilli()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + refreshFn := func(fnCtx context.Context) (int64, error) { + select { + case <-time.After(2 * time.Second): + return 42, nil + case <-fnCtx.Done(): + return 0, fnCtx.Err() + } + } + + value, err := counter.TryUpdateIfStale(ctx, time.Second, refreshFn) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, int64(5), value, "foreground returns the current value once updateMaxWait passes") + + assert.Eventually(t, func() bool { return counter.GetValue() == 42 }, 4*time.Second, 20*time.Millisecond, + "result delivered after fallbackTimeout but within the caller's deadline must be applied") +} diff --git a/util/gzip_pool.go b/util/gzip_pool.go index ee4649713..22d410ffb 100644 --- a/util/gzip_pool.go +++ b/util/gzip_pool.go @@ -53,11 +53,14 @@ func (p *GzipReaderPool) Put(zr *gzip.Reader) { p.pool.Put(zr) } -// pooledGzipReadCloser wraps a gzip.Reader so that closing it returns it to the pool. +// pooledGzipReadCloser wraps a gzip.Reader so that closing it returns it to the +// pool and closes the source. gzip.Reader.Close does not close its source, so +// an HTTP response body would otherwise keep its stream open. type pooledGzipReadCloser struct { - zr *gzip.Reader - pool *GzipReaderPool - once sync.Once + zr *gzip.Reader + pool *GzipReaderPool + source io.Closer + once sync.Once } func (pgrc *pooledGzipReadCloser) Read(b []byte) (int, error) { return pgrc.zr.Read(b) } @@ -68,16 +71,23 @@ func (pgrc *pooledGzipReadCloser) Close() error { // Close underlying gzip reader first, then return to pool exactly once. err = pgrc.zr.Close() pgrc.pool.Put(pgrc.zr) + if pgrc.source != nil { + if cerr := pgrc.source.Close(); cerr != nil && err == nil { + err = cerr + } + } // Clear references to avoid accidental reuse and help GC pgrc.zr = nil pgrc.pool = nil + pgrc.source = nil }) return err } // WrapGzipReader returns an io.ReadCloser wrapper that will return the gzip.Reader to pool on Close. -func (p *GzipReaderPool) WrapGzipReader(zr *gzip.Reader) io.ReadCloser { - return &pooledGzipReadCloser{zr: zr, pool: p} +// A non-nil source is closed too; pass nil when the caller closes it. +func (p *GzipReaderPool) WrapGzipReader(zr *gzip.Reader, source io.Closer) io.ReadCloser { + return &pooledGzipReadCloser{zr: zr, pool: p, source: source} } // GzipWriterPool wraps a sync.Pool for gzip.Writer with helpers to reset writers diff --git a/util/gzip_pool_close_test.go b/util/gzip_pool_close_test.go new file mode 100644 index 000000000..84419fbea --- /dev/null +++ b/util/gzip_pool_close_test.go @@ -0,0 +1,130 @@ +package util + +import ( + "bytes" + "compress/gzip" + "errors" + "io" + "sync" + "sync/atomic" + "testing" +) + +// countingCloser tracks Close call count + records prior Close error if any. +type countingCloser struct { + io.Reader + closes atomic.Int32 + closeErr error +} + +func (c *countingCloser) Close() error { + c.closes.Add(1) + return c.closeErr +} + +func gzipped(t *testing.T, payload string) io.Reader { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write([]byte(payload)); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return &buf +} + +// pooledGzipReadCloser.Close must close the underlying source. +func TestPooledGzipReadCloser_ClosesSource(t *testing.T) { + pool := NewGzipReaderPool() + src := &countingCloser{Reader: gzipped(t, "hello")} + + zr, err := pool.GetReset(src) + if err != nil { + t.Fatalf("GetReset: %v", err) + } + rc := pool.WrapGzipReader(zr, src) + + // Drain + if _, err := io.ReadAll(rc); err != nil { + t.Fatalf("read: %v", err) + } + + if err := rc.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + if got := src.closes.Load(); got != 1 { + t.Fatalf("expected source.Close called exactly once, got %d", got) + } +} + +// Close must be idempotent across many goroutines and only Close the source +// once even under contention. +func TestPooledGzipReadCloser_CloseIdempotent(t *testing.T) { + pool := NewGzipReaderPool() + src := &countingCloser{Reader: gzipped(t, "hello")} + + zr, err := pool.GetReset(src) + if err != nil { + t.Fatalf("GetReset: %v", err) + } + rc := pool.WrapGzipReader(zr, src) + + if _, err := io.ReadAll(rc); err != nil { + t.Fatalf("read: %v", err) + } + + var wg sync.WaitGroup + for range 32 { + wg.Add(1) + go func() { defer wg.Done(); _ = rc.Close() }() + } + wg.Wait() + + if got := src.closes.Load(); got != 1 { + t.Fatalf("expected source.Close called exactly once even under contention, got %d", got) + } +} + +// Source is optional — passing nil is supported (some callers manage the +// source lifetime themselves) and Close should not panic. +func TestPooledGzipReadCloser_NilSourceOk(t *testing.T) { + pool := NewGzipReaderPool() + src := gzipped(t, "hello") + + zr, err := pool.GetReset(src) + if err != nil { + t.Fatalf("GetReset: %v", err) + } + rc := pool.WrapGzipReader(zr, nil) + + if _, err := io.ReadAll(rc); err != nil { + t.Fatalf("read: %v", err) + } + if err := rc.Close(); err != nil { + t.Fatalf("close with nil source: %v", err) + } +} + +// If the source's Close errors, the wrapper surfaces that error (preferring +// the gzip-reader error if both fail, since gzip is closed first). +func TestPooledGzipReadCloser_PropagatesSourceError(t *testing.T) { + pool := NewGzipReaderPool() + wantErr := errors.New("source close failed") + src := &countingCloser{Reader: gzipped(t, "hello"), closeErr: wantErr} + + zr, err := pool.GetReset(src) + if err != nil { + t.Fatalf("GetReset: %v", err) + } + rc := pool.WrapGzipReader(zr, src) + if _, err := io.ReadAll(rc); err != nil { + t.Fatalf("read: %v", err) + } + + if err := rc.Close(); !errors.Is(err, wantErr) { + t.Fatalf("expected source close error to propagate, got %v", err) + } +} diff --git a/util/initializer.go b/util/initializer.go index 5d2a88e12..ab1396644 100644 --- a/util/initializer.go +++ b/util/initializer.go @@ -53,6 +53,9 @@ type BootstrapTask struct { ctxCancel atomic.Value // context.CancelFunc doneVal atomic.Value // chan struct{} attempts atomic.Int32 + // attemptMu makes "claim Running + bump attempts" and "check attempt id + + // leave Running" atomic, so a superseded attempt cannot finish the next one. + attemptMu sync.Mutex } func NewBootstrapTask(name string, fn func(ctx context.Context) error) *BootstrapTask { @@ -106,6 +109,12 @@ func (t *BootstrapTask) Wait(ctx context.Context) error { if ok && lastErr.err != nil { return lastErr.err } + if state != TaskSucceeded { + // The terminal state is set before its error is stored, so a + // failure can be observed with no error yet. Never report it + // as success. + return fmt.Errorf("task %s %s without a recorded error", t.Name, state) + } return nil } ch := t.doneVal.Load() @@ -129,24 +138,41 @@ func (t *BootstrapTask) Wait(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ch.(chan struct{}): - // The attempt ended. Check if we failed. - if TaskState(t.state.Load()) == TaskFailed { - wr, _ := t.lastErr.Load().(wrappedError) - if wr.err == nil { - t.lastErr.Store(wrappedError{err: errors.New("task failed without specific error")}) - } - return wr.err - } - return nil // Succeeded or otherwise finished + // The attempt ended; re-check the terminal state so every + // failure kind surfaces lastErr. + continue } } } } -// attempt is called just before a new attempt to run t.Fn. -func (t *BootstrapTask) beginAttempt() { - t.attempts.Add(1) +// startAttempt moves the task from state `from` to Running and returns the new +// attempt's id, or false when another caller changed the state first. +func (t *BootstrapTask) startAttempt(from TaskState) (int32, bool) { + t.attemptMu.Lock() + defer t.attemptMu.Unlock() + // #nosec G115 - We know TaskState is small enough that int->int32 won't overflow + if !t.state.CompareAndSwap(int32(from), int32(TaskRunning)) { + return 0, false + } t.lastAttempt.Store(time.Now()) + return t.attempts.Add(1), true +} + +// finishAttempt applies terminal only while attemptID is still the current, +// Running attempt, so a reaped or superseded Fn cannot clobber a later retry. +func (t *BootstrapTask) finishAttempt(attemptID int32, terminal TaskState, err error) bool { + t.attemptMu.Lock() + defer t.attemptMu.Unlock() + if t.attempts.Load() != attemptID { + return false + } + // #nosec G115 - We know TaskState is small enough that int->int32 won't overflow + if !t.state.CompareAndSwap(int32(TaskRunning), int32(terminal)) { + return false + } + t.lastErr.Store(wrappedError{err: err}) + return true } type InitializerConfig struct { @@ -224,24 +250,54 @@ func (i *Initializer) WaitForTasks(ctx context.Context) error { return i.waitForTasks(ctx, allTasks...) } -// Wait for a set of tasks to complete or ctx to expire. +// Wait for a set of tasks to complete or ctx to expire. Waits run in +// parallel so one hung task cannot consume the budget of the others. func (i *Initializer) waitForTasks(ctx context.Context, tasks ...*BootstrapTask) error { - var errs []error + if len(tasks) == 0 { + return nil + } + + type waitResult struct { + task *BootstrapTask + err error + } + errCh := make(chan waitResult, len(tasks)) for _, task := range tasks { - if err := task.Wait(ctx); err != nil { - // If context was canceled, likely best to just return. - return err + go func(task *BootstrapTask) { + errCh <- waitResult{task: task, err: task.Wait(ctx)} + }(task) + } + + var errs []error + var ctxErr error + for range tasks { + res := <-errCh + if res.err == nil { + continue } - // If task is failed, record that error - state := TaskState(task.state.Load()) - if state == TaskFailed && task.Error() != nil { - errs = append(errs, task.Error().Err) + st := TaskState(res.task.state.Load()) + // A context error for a task still in flight means the wait itself + // was aborted; a finished TimedOut task is a task failure. + if (errors.Is(res.err, context.Canceled) || errors.Is(res.err, context.DeadlineExceeded)) && + (st == TaskPending || st == TaskRunning) { + if ctxErr == nil { + ctxErr = res.err + } + continue } + errs = append(errs, res.err) + } + if ctxErr != nil { + return ctxErr } if len(errs) > 0 { total := len(tasks) i.logger.Warn().Errs("tasks", errs).Msgf("initialization failed: %d/%d tasks failed", len(errs), total) - return fmt.Errorf("initialization failed: %d/%d tasks failed: %v", len(errs), total, errs) + // Keep the tasks' own errors reachable for typed error checks. + if len(errs) == 1 { + return errs[0] + } + return errors.Join(append([]error{fmt.Errorf("initialization failed: %d/%d tasks failed", len(errs), total)}, errs...)...) } return nil } @@ -320,24 +376,22 @@ func (i *Initializer) attemptRemainingTasks(respectBackoff bool) { return true } // Attempt to swap from [Pending|Failed|Timeout] -> Running - // #nosec G115 - We know TaskState is small enough that int->int32 won't overflow - if t.state.CompareAndSwap(int32(state), int32(TaskRunning)) { - t.beginAttempt() + if attemptID, ok := t.startAttempt(state); ok { t.lastErr.Store(wrappedError{err: nil}) // Create a fresh done channel to signal this attempt's completion doneCh := t.createNewDoneChannel() tasksToRun = append(tasksToRun, t) - go func(bt *BootstrapTask, doneCh chan struct{}) { + go func(bt *BootstrapTask, doneCh chan struct{}, attemptID int32) { // Close the channel when the function finishes - // The CompareAndSwap will ensure we always and only close the channel once for each attempt defer close(doneCh) if i.appCtx.Err() != nil { - bt.lastErr.Store(wrappedError{err: i.appCtx.Err()}) - bt.state.Store(int32(TaskFailed)) - i.logger.Warn().Str("task", bt.Name).Err(i.appCtx.Err()).Msg("initialization task context error") + if bt.finishAttempt(attemptID, TaskFailed, i.appCtx.Err()) { + i.logger.Warn().Str("task", bt.Name).Err(i.appCtx.Err()).Msg("initialization task context error") + } + wg.Done() return } @@ -360,30 +414,35 @@ func (i *Initializer) attemptRemainingTasks(respectBackoff bool) { if uw, ok := err.(interface{ Unwrap() error }); ok && uw.Unwrap() != nil { underlying = uw.Unwrap() } - bt.lastErr.Store(wrappedError{err: underlying}) - bt.state.Store(int32(TaskFatal)) - // Log the underlying fatal error - i.logger.Error().Str("task", bt.Name).Err(underlying).Msg("initialization task fatal error") + if bt.finishAttempt(attemptID, TaskFatal, underlying) { + i.logger.Error().Str("task", bt.Name).Err(underlying).Msg("initialization task fatal error") + } return } - // If context is cancelled there will be a reason already set for it on lastErr if !errors.Is(err, context.Canceled) { if cause := context.Cause(tctx); cause != nil { err = cause } - bt.lastErr.Store(wrappedError{err: err}) } else { - bt.lastErr.CompareAndSwap(nil, wrappedError{err: err}) + // Preserve a reason already set (e.g. by reap) when canceled. + if wr, ok := bt.lastErr.Load().(wrappedError); ok && wr.err != nil { + err = wr.err + } + } + terminal := TaskFailed + if errors.Is(err, context.DeadlineExceeded) { + terminal = TaskTimedOut + } + if bt.finishAttempt(attemptID, terminal, err) { + i.logger.Warn().Str("task", bt.Name).Err(err).Str("state", terminal.String()).Msg("initialization task failed") } - bt.state.Store(int32(TaskFailed)) - i.logger.Warn().Str("task", bt.Name).Err(err).Msg("initialization task failed") } else { - bt.lastErr.Store(wrappedError{err: nil}) - bt.state.Store(int32(TaskSucceeded)) - lastAttempt, _ := bt.lastAttempt.Load().(time.Time) - i.logger.Info().Str("task", bt.Name).Dur("durationMs", time.Since(lastAttempt)).Msg("initialization task succeeded") + if bt.finishAttempt(attemptID, TaskSucceeded, nil) { + lastAttempt, _ := bt.lastAttempt.Load().(time.Time) + i.logger.Info().Str("task", bt.Name).Dur("durationMs", time.Since(lastAttempt)).Msg("initialization task succeeded") + } } - }(t, doneCh) + }(t, doneCh, attemptID) } else { wg.Done() } @@ -398,7 +457,7 @@ func (i *Initializer) attemptRemainingTasks(respectBackoff bool) { } func (i *Initializer) State() InitializationState { - var total, pending, running, succeeded, failed, fatal int + var total, pending, running, succeeded, failed, timedOut, fatal int i.tasks.Range(func(key, value interface{}) bool { t := value.(*BootstrapTask) state := TaskState(t.state.Load()) @@ -411,6 +470,8 @@ func (i *Initializer) State() InitializationState { succeeded++ case TaskFailed: failed++ + case TaskTimedOut: + timedOut++ case TaskFatal: fatal++ } @@ -424,31 +485,76 @@ func (i *Initializer) State() InitializationState { Int("running", running). Int("succeeded", succeeded). Int("failed", failed). + Int("timedOut", timedOut). Int("fatal", fatal). Msg("calculating initialization state") + if total == 0 { + return StateUninitialized + } if total == succeeded { return StateReady } - // If any fatal exists, prefer Fatal state - if fatal > 0 { - return StateFatal - } - // If all tasks are done (some are failed, none running or pending), it's a "Failed" state - if failed > 0 && (pending+running+succeeded == 0) { + + // A fatal task must not mark the whole initializer fatal while other + // tasks can still recover. + retryable := failed + timedOut + inFlight := pending + running + nonTerminal := inFlight + retryable + + if nonTerminal > 0 { + atp := i.attempts.Load() + if atp > 1 { + return StateRetrying + } + if inFlight > 0 { + return StateInitializing + } + // Only retryable tasks left (awaiting auto-retry), nothing in-flight. + if succeeded > 0 || fatal > 0 { + return StatePartial + } return StateFailed } - if failed > 0 && (pending+running == 0) { - return StatePartial + + // All terminal: succeeded and/or fatal only. + if fatal == total { + return StateFatal } - // If we've tried multiple times but still have tasks not succeeded - atp := i.attempts.Load() - if atp > 1 && (pending > 0 || running > 0) { - return StateRetrying + if fatal > 0 { + return StatePartial } return StateInitializing } +// reapOverdueRunningTasks moves Running tasks whose attempt exceeded +// TaskTimeout to TaskTimedOut, so a Fn that ignores its ctx leaves the task +// retryable instead of Running forever. +func (i *Initializer) reapOverdueRunningTasks() { + now := time.Now() + i.tasks.Range(func(_, value interface{}) bool { + t := value.(*BootstrapTask) + if TaskState(t.state.Load()) != TaskRunning { + return true + } + lastAttempt, _ := t.lastAttempt.Load().(time.Time) + if lastAttempt.IsZero() || now.Sub(lastAttempt) < i.conf.TaskTimeout { + return true + } + if cancel, ok := t.ctxCancel.Load().(context.CancelFunc); ok && cancel != nil { + cancel() + } + if t.state.CompareAndSwap(int32(TaskRunning), int32(TaskTimedOut)) { + t.lastErr.Store(wrappedError{err: context.DeadlineExceeded}) + i.logger.Warn(). + Str("task", t.Name). + Dur("runningFor", now.Sub(lastAttempt)). + Msg("initialization task timed out while still running; reaped for retry") + } + return true + }) +} + func (i *Initializer) Status() *InitializerStatus { state := i.State() return &InitializerStatus{ @@ -492,16 +598,16 @@ func (i *Initializer) MarkTaskAsFailed(name string, err error) { func (i *Initializer) Stop(destroyFn func() error) error { i.logger.Debug().Msg("stopping initializer") - i.tasksMu.Lock() - defer i.tasksMu.Unlock() - + // Stop the auto-retry loop before taking tasksMu, which the loop also + // acquires. if cancel := i.cancelAutoRetry.Load(); cancel != nil { cancel.(context.CancelFunc)() } - - // Wait for auto-retry goroutine to finish i.autoRetryWg.Wait() + i.tasksMu.Lock() + defer i.tasksMu.Unlock() + // Now, wait for any tasks that might still be running to finish or fail. waitCtx, waitCancel := context.WithTimeout(i.appCtx, i.conf.TaskTimeout+100*time.Millisecond) defer waitCancel() @@ -659,7 +765,12 @@ func (i *Initializer) autoRetryLoop(ctx context.Context) { } i.attempts.Add(1) i.attemptRemainingTasks(false) - err := i.WaitForTasks(ctx) + // Bound the wait so a Fn that ignores its ctx cannot stop retries of + // every other task. + waitCtx, waitCancel := context.WithTimeout(ctx, i.conf.TaskTimeout) + err := i.WaitForTasks(waitCtx) + waitCancel() + i.reapOverdueRunningTasks() state := i.State() // Stop only once no task can benefit from another attempt (every task // succeeded or is fatal). Fatal tasks are skipped by diff --git a/util/initializer_test.go b/util/initializer_test.go index 4a3fe4dd8..847a739c8 100644 --- a/util/initializer_test.go +++ b/util/initializer_test.go @@ -339,7 +339,7 @@ func TestInitializer_TaskTimeout(t *testing.T) { defer init.Stop(nil) require.Error(t, err) assert.Equal(t, StateFailed, init.State()) - assert.Equal(t, TaskFailed, TaskState(task.state.Load())) + assert.Equal(t, TaskTimedOut, TaskState(task.state.Load())) assert.ErrorIs(t, task.Error().Err, context.DeadlineExceeded) } @@ -442,11 +442,13 @@ func TestInitializer_MultipleRapidFailures(t *testing.T) { // Check we tried multiple times (rapidly) assert.True(t, attempts > 1, "should attempt multiple times in quick succession") - // Check final State is either partial or failed + // Auto-retry is still armed, so aggregate state is Retrying while the + // sole task keeps failing between attempts (Failed only once the loop + // has stopped and nothing remains in-flight). state := init.State() assert.True( t, - state == StateFailed, + state == StateFailed || state == StateRetrying, "final state should reflect the repeated failures, got %v", state, ) @@ -484,9 +486,9 @@ func TestInitializer_ForcedCancellationMidTask(t *testing.T) { require.Error(t, err, "should fail or be canceled") assert.ErrorIs(t, err, context.DeadlineExceeded) - // Check task state is failed (or timed out) after forced cancel + // ExecuteTasks already ran until TaskTimeout; deadline surfaces as TimedOut. st := TaskState(task.state.Load()) - assert.True(t, st == TaskFailed, "task should show failed or timed out, got %d", st) + assert.True(t, st == TaskFailed || st == TaskTimedOut, "task should show failed or timed out, got %d", st) } func TestInitializer_MarkTaskAsFailedMidRun(t *testing.T) { @@ -1065,3 +1067,241 @@ func TestInitializer_FatalTaskWithSucceededSiblingExits(t *testing.T) { t.Fatal("Stop did not return promptly after all tasks reached terminal states") } } + +// Regression: tasks scheduled after all earlier tasks succeeded must still +// be retried (the loop must not have wound down for good). +func TestInitializer_RetryLoopRestartsForNewFailedTasks(t *testing.T) { + conf := &InitializerConfig{ + TaskTimeout: time.Second, + AutoRetry: true, + RetryMinDelay: time.Millisecond * 10, + RetryMaxDelay: time.Millisecond * 20, + RetryFactor: 1.2, + } + appCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + init := setupInitializer(t, appCtx, conf) + + ok := NewBootstrapTask("ok-task", func(ctx context.Context) error { return nil }) + _ = init.ExecuteTasks(appCtx, ok) + require.Eventually(t, func() bool { + return TaskState(ok.state.Load()) == TaskSucceeded + }, 5*time.Second, 10*time.Millisecond, "first task should succeed") + + var attempts atomic.Int32 + transientTask := NewBootstrapTask("late-transient-task", func(ctx context.Context) error { + if attempts.Add(1) < 3 { + return errors.New("transient failure") + } + return nil + }) + _ = init.ExecuteTasks(appCtx, transientTask) + + require.Eventually(t, func() bool { + return TaskState(transientTask.state.Load()) == TaskSucceeded + }, 5*time.Second, 20*time.Millisecond, "late task should be retried by a (re)started loop") + + init.Stop(nil) +} + +// Regression: a task whose Fn hangs (ignores ctx) must not block the +// auto-retry loop for other, transiently-failing tasks. +func TestInitializer_HungTaskDoesNotBlockRetryOfOthers(t *testing.T) { + conf := &InitializerConfig{ + TaskTimeout: time.Millisecond * 200, + AutoRetry: true, + RetryMinDelay: time.Millisecond * 10, + RetryMaxDelay: time.Millisecond * 20, + RetryFactor: 1.2, + } + appCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + init := setupInitializer(t, appCtx, conf) + + hungRelease := make(chan struct{}) + hungTask := NewBootstrapTask("hung-task", func(ctx context.Context) error { + <-hungRelease // ignores ctx: simulates a client dial with no deadline + return nil + }) + + var attempts atomic.Int32 + transientTask := NewBootstrapTask("transient-task", func(ctx context.Context) error { + if attempts.Add(1) < 3 { + return errors.New("transient failure") + } + return nil + }) + + go func() { _ = init.ExecuteTasks(appCtx, hungTask, transientTask) }() + + require.Eventually(t, func() bool { + return TaskState(transientTask.state.Load()) == TaskSucceeded + }, 5*time.Second, 20*time.Millisecond, "transient task should eventually succeed despite the hung sibling") + + // Hung attempt must be reaped to TimedOut (retryable), not left Running forever. + require.Eventually(t, func() bool { + st := TaskState(hungTask.state.Load()) + return st == TaskTimedOut || st == TaskRunning // Running only during a brief retry window + }, 2*time.Second, 20*time.Millisecond) + + // Observe at least one TimedOut transition (reap happened). + require.Eventually(t, func() bool { + return TaskState(hungTask.state.Load()) == TaskTimedOut || + hungTask.attempts.Load() >= 2 // reaped and retried + }, 3*time.Second, 20*time.Millisecond, "hung task should be reaped (TimedOut) and/or retried") + + close(hungRelease) + init.Stop(nil) +} + +func TestInitializer_State_FatalSiblingWithSuccessIsPartial(t *testing.T) { + conf := &InitializerConfig{ + TaskTimeout: time.Second, + AutoRetry: false, + } + appCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + init := setupInitializer(t, appCtx, conf) + + fatalTask := NewBootstrapTask("fatal-task", func(ctx context.Context) error { + return &testFatalError{errors.New("permanent misconfiguration")} + }) + okTask := NewBootstrapTask("ok-task", func(ctx context.Context) error { return nil }) + + _ = init.ExecuteTasks(appCtx, fatalTask, okTask) + require.Equal(t, TaskFatal, TaskState(fatalTask.state.Load())) + require.Equal(t, TaskSucceeded, TaskState(okTask.state.Load())) + assert.Equal(t, StatePartial, init.State(), + "one fatal sibling must not mark the whole initializer Fatal when others succeeded") + init.Stop(nil) +} + +func TestInitializer_State_AllFatalIsFatal(t *testing.T) { + conf := &InitializerConfig{ + TaskTimeout: time.Second, + AutoRetry: false, + } + appCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + init := setupInitializer(t, appCtx, conf) + + a := NewBootstrapTask("fatal-a", func(ctx context.Context) error { + return &testFatalError{errors.New("bad a")} + }) + b := NewBootstrapTask("fatal-b", func(ctx context.Context) error { + return &testFatalError{errors.New("bad b")} + }) + _ = init.ExecuteTasks(appCtx, a, b) + assert.Equal(t, StateFatal, init.State()) + init.Stop(nil) +} + +func TestInitializer_WaitForTasks_ParallelDoesNotSerializeOnSlowSibling(t *testing.T) { + conf := &InitializerConfig{ + TaskTimeout: time.Second, + AutoRetry: false, + } + appCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + init := setupInitializer(t, appCtx, conf) + + slowStarted := make(chan struct{}) + slow := NewBootstrapTask("slow", func(ctx context.Context) error { + close(slowStarted) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + return nil + } + }) + fast := NewBootstrapTask("fast", func(ctx context.Context) error { return nil }) + + go func() { _ = init.ExecuteTasks(appCtx, slow, fast) }() + <-slowStarted + + // Short wait: with sequential Wait, a slow-first Range order could burn the + // whole budget before observing fast. Parallel wait must see fast succeed. + waitCtx, waitCancel := context.WithTimeout(appCtx, 100*time.Millisecond) + defer waitCancel() + _ = init.WaitForTasks(waitCtx) + + assert.Equal(t, TaskSucceeded, TaskState(fast.state.Load()), "fast task must complete even while slow is in-flight") + init.Stop(nil) +} + +type testFatalError struct{ error } + +func (e *testFatalError) IsTaskFatal() bool { return true } +func (e *testFatalError) Unwrap() error { return e.error } + +// A reaped attempt whose Fn returns while the next attempt is starting must +// never finish that next attempt. +func TestBootstrapTask_SupersededAttemptCannotFinishNextAttempt(t *testing.T) { + for i := 0; i < 20000; i++ { + bt := NewBootstrapTask("t", nil) + bt.attempts.Store(1) + bt.state.Store(int32(TaskTimedOut)) // attempt 1 was reaped + + var started bool + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + _, started = bt.startAttempt(TaskTimedOut) + }() + go func() { + defer wg.Done() + bt.finishAttempt(1, TaskFailed, errors.New("late")) + }() + wg.Wait() + + require.True(t, started) + require.Equal(t, TaskRunning, TaskState(bt.state.Load()), "iteration %d", i) + } +} + +// A failed attempt sets its terminal state before storing its error, so Wait +// can observe the state while lastErr still holds the attempt-start nil. It +// must still report a failure. +func TestBootstrapTask_WaitTerminalFailureWithoutErrorIsError(t *testing.T) { + for _, st := range []TaskState{TaskFailed, TaskTimedOut, TaskFatal} { + bt := NewBootstrapTask("t", nil) + bt.lastErr.Store(wrappedError{err: nil}) + bt.state.Store(int32(st)) + require.Error(t, bt.Wait(context.Background()), st.String()) + } + + bt := NewBootstrapTask("ok", nil) + bt.lastErr.Store(wrappedError{err: nil}) + bt.state.Store(int32(TaskSucceeded)) + require.NoError(t, bt.Wait(context.Background())) +} + +// Once the app context is cancelled, an attempt ends before running its task. +// It must still release the attempt WaitGroup, which ExecuteTasks waits on +// while holding tasksMu; otherwise every later call, and Stop, hang. +func TestInitializer_CancelledAppCtxDoesNotWedge(t *testing.T) { + appCtx, cancel := context.WithCancel(context.Background()) + init := setupInitializer(t, appCtx, &InitializerConfig{TaskTimeout: time.Second}) + cancel() + + done := make(chan error, 1) + go func() { + done <- init.ExecuteTasks(context.Background(), NewBootstrapTask("t", func(context.Context) error { return nil })) + }() + select { + case err := <-done: + require.Error(t, err) + case <-time.After(2 * time.Second): + t.Fatal("ExecuteTasks hung after the app context was cancelled") + } + + stopped := make(chan struct{}) + go func() { _ = init.Stop(nil); close(stopped) }() + select { + case <-stopped: + case <-time.After(2 * time.Second): + t.Fatal("Stop hung on tasksMu") + } +} diff --git a/util/testing.go b/util/testing.go index e18f6ada7..95aa213d0 100644 --- a/util/testing.go +++ b/util/testing.go @@ -360,6 +360,8 @@ func ResetGock() { gock.CleanUnmatchedRequest() gock.Disable() + // Networking filters accumulate across tests unless cleared. + gock.DisableNetworkingFilters() gock.EnableNetworking() gock.NetworkingFilter(func(req *http.Request) bool { host := strings.Split(req.URL.Host, ":")[0] From a8718b51942955b23299040386c9a6cd955d00bf Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 13:27:41 +0100 Subject: [PATCH 02/21] feat(config): WebSocket server, indexer and failover configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - server.webSocket: buffer and message sizes, ping interval, per- connection subscription and in-flight request limits, and the per- subscription notification buffer; validated. - indexer: dedupWindowSize (canonicalChainDepth is accepted and ignored). - networks[].failover / networkDefaults.failover: onDefaultsExhausted, with a validation-report warning when no upstream is tagged tier:fallback. - evm.stripSubscribeFromBlockZero, inherited from networkDefaults. - Metrics for WebSocket connectivity, dropped subscription notifications and the fallback escape; generated TS types. Co-authored-by: snowkide Co-authored-by: João Gomes Co-Authored-By: Claude Opus 5.5 --- common/config.go | 63 +++++++++++++++++++++ common/config_test.go | 85 +++++++++++++++++++++++++++++ common/defaults.go | 42 +++++++++++++- common/errors.go | 64 ++++++++++++++++++++++ common/request.go | 13 +++++ common/validation.go | 31 +++++++++++ common/validation_websocket_test.go | 38 +++++++++++++ erpc/config_analyzer.go | 46 ++++++++++++++-- erpc/config_analyzer_test.go | 42 ++++++++++++++ telemetry/metrics.go | 21 +++++++ typescript/config/src/generated.ts | 63 +++++++++++++++++++++ 11 files changed, 502 insertions(+), 6 deletions(-) create mode 100644 common/validation_websocket_test.go create mode 100644 erpc/config_analyzer_test.go diff --git a/common/config.go b/common/config.go index b2f75d177..3e68293bc 100644 --- a/common/config.go +++ b/common/config.go @@ -47,6 +47,7 @@ type Config struct { Projects []*ProjectConfig `yaml:"projects,omitempty" json:"projects"` RateLimiters *RateLimiterConfig `yaml:"rateLimiters,omitempty" json:"rateLimiters"` Metrics *MetricsConfig `yaml:"metrics,omitempty" json:"metrics"` + Indexer *IndexerConfig `yaml:"indexer,omitempty" json:"indexer"` ProxyPools []*ProxyPoolConfig `yaml:"proxyPools,omitempty" json:"proxyPools"` Tracing *TracingConfig `yaml:"tracing,omitempty" json:"tracing"` @@ -84,6 +85,16 @@ var LegacyTranslateFn func(*Config) ([]string, error) // emitted by LegacyTranslateFn. If nil, warnings are dropped silently. var LegacyTranslateLogger func(warning string) +// IndexerConfig tunes the event-stream indexer behind eth_subscribe fan-out. +type IndexerConfig struct { + // Deprecated: ignored; the indexer no longer tracks a canonical chain. + // Kept so configs that still set it load under strict decoding. + CanonicalChainDepth int `yaml:"canonicalChainDepth,omitempty" json:"canonicalChainDepth"` + // DedupWindowSize is the per-filter seen-set capacity for log and + // pending-tx fan-out across upstreams. 0 uses the default (8192). + DedupWindowSize int `yaml:"dedupWindowSize,omitempty" json:"dedupWindowSize"` +} + // LoadConfig loads the configuration from the specified file. // It supports both YAML and TypeScript (.ts) files. func LoadConfig(fs afero.Fs, filename string, opts *DefaultOptions) (*Config, error) { @@ -162,6 +173,8 @@ type ServerConfig struct { TrustedIPHeaders []string `yaml:"trustedIPHeaders,omitempty" json:"trustedIPHeaders"` ResponseHeaders map[string]string `yaml:"responseHeaders,omitempty" json:"responseHeaders"` + WebSocket *WebSocketServerConfig `yaml:"webSocket,omitempty" json:"webSocket"` + // ExecutionHeaders controls the per-request diagnostic headers // (X-ERPC-Attempts, X-ERPC-Upstreams-Tried, etc.) that expose how // eRPC routed and resolved each request. Defaults to "all" — set @@ -177,6 +190,23 @@ type ServerConfig struct { CostHeaders *bool `yaml:"costHeaders,omitempty" json:"costHeaders"` } +type WebSocketServerConfig struct { + ReadBufferSize int `yaml:"readBufferSize,omitempty" json:"readBufferSize"` + WriteBufferSize int `yaml:"writeBufferSize,omitempty" json:"writeBufferSize"` + MaxMessageSize int64 `yaml:"maxMessageSize,omitempty" json:"maxMessageSize"` + PingInterval *Duration `yaml:"pingInterval,omitempty" json:"pingInterval" tstype:"Duration"` + MaxSubscriptionsPerConnection int `yaml:"maxSubscriptionsPerConnection,omitempty" json:"maxSubscriptionsPerConnection"` + // MaxConcurrentRequestsPerConnection bounds the messages a connection has + // in flight; further frames wait (backpressure) instead of being rejected. + // A batch is one message; its requests run concurrently, as over HTTP. + MaxConcurrentRequestsPerConnection int `yaml:"maxConcurrentRequestsPerConnection,omitempty" json:"maxConcurrentRequestsPerConnection"` + // SubscriptionBufferSize is the number of notifications queued per + // subscription for a client that is behind. On overflow newHeads and + // newPendingTransactions drop their oldest entry; logs close the + // connection instead (1013), so the client knows it missed data. + SubscriptionBufferSize int `yaml:"subscriptionBufferSize,omitempty" json:"subscriptionBufferSize"` +} + // ExecutionHeadersMode controls how much per-request execution detail is // exposed in HTTP response headers. type ExecutionHeadersMode string @@ -716,6 +746,24 @@ type NetworkDefaults struct { Svm *SvmNetworkConfig `yaml:"svm,omitempty" json:"svm" tstype:"TsSvmNetworkConfigForDefaults"` Multiplexing *bool `yaml:"multiplexing,omitempty" json:"multiplexing"` CacheKeySuffix string `yaml:"cacheKeySuffix,omitempty" json:"cacheKeySuffix"` + Failover *FailoverConfig `yaml:"failover,omitempty" json:"failover"` +} + +// FailoverConfig controls per-request escalation to fallback-tier upstreams, +// independent of the selection policy. +type FailoverConfig struct { + // OnDefaultsExhausted tries upstreams not tagged `tier:fallback` first and + // advances to the fallback tier within the same request only once every + // other upstream failed with an error retryable toward the network. + OnDefaultsExhausted *bool `yaml:"onDefaultsExhausted,omitempty" json:"onDefaultsExhausted"` +} + +// Enabled reports whether any failover behaviour is configured. +func (f *FailoverConfig) Enabled() bool { + if f == nil { + return false + } + return f.OnDefaultsExhausted != nil && *f.OnDefaultsExhausted } // UnmarshalYAML provides backward compatibility for old single failsafe object format @@ -750,6 +798,7 @@ func (n *NetworkDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error Evm *EvmNetworkConfig `yaml:"evm,omitempty"` Svm *SvmNetworkConfig `yaml:"svm,omitempty"` CacheKeySuffix string `yaml:"cacheKeySuffix,omitempty"` + Failover *FailoverConfig `yaml:"failover,omitempty"` } var old oldNetworkDefaults @@ -766,6 +815,7 @@ func (n *NetworkDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error n.DirectiveDefaults = old.DirectiveDefaults n.Evm = old.Evm n.Svm = old.Svm + n.Failover = old.Failover if old.Failsafe != nil { // Ensure MatchMethod has a default value for backward compatibility @@ -849,6 +899,10 @@ func (p *ProviderConfig) MarshalYAML() (interface{}, error) { }, nil } +// TagTierFallback marks an upstream as part of the fallback tier, used only +// when the other upstreams are unavailable. +const TagTierFallback = "tier:fallback" + // RateLimitCountMode selects the accounting unit an upstream's rate-limit // budget charges per call. type RateLimitCountMode string @@ -2279,6 +2333,7 @@ type NetworkConfig struct { Methods *MethodsConfig `yaml:"methods,omitempty" json:"methods"` Multiplexing *bool `yaml:"multiplexing,omitempty" json:"multiplexing"` StaticResponses []*StaticResponseConfig `yaml:"staticResponses,omitempty" json:"staticResponses,omitempty"` + Failover *FailoverConfig `yaml:"failover,omitempty" json:"failover"` // Integrity overrides the project-wide data-integrity configuration for this // network. Merges over the project block (network wins). Integrity *IntegrityConfig `yaml:"integrity,omitempty" json:"integrity,omitempty"` @@ -2357,6 +2412,7 @@ func (n *NetworkConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { Methods *MethodsConfig `yaml:"methods,omitempty"` StaticResponses []*StaticResponseConfig `yaml:"staticResponses,omitempty"` CacheKeySuffix string `yaml:"cacheKeySuffix,omitempty"` + Failover *FailoverConfig `yaml:"failover,omitempty"` } var old oldNetworkConfig @@ -2377,6 +2433,7 @@ func (n *NetworkConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { n.Methods = old.Methods n.StaticResponses = old.StaticResponses n.CacheKeySuffix = old.CacheKeySuffix + n.Failover = old.Failover if old.Failsafe != nil { // Ensure MatchMethod has a default value for backward compatibility @@ -2584,6 +2641,12 @@ type EvmNetworkConfig struct { // Default includes common point-lookup methods like eth_getBlockByNumber, eth_getTransactionByHash, etc. MarkEmptyAsErrorMethods []string `yaml:"markEmptyAsErrorMethods,omitempty" json:"markEmptyAsErrorMethods,omitempty"` + // StripSubscribeFromBlockZero removes a zero `fromBlock` from eth_subscribe + // logs filters before forwarding upstream. fromBlock has no meaning for a + // live subscription, and backends that prune history reject it. + // DEFAULT: false. + StripSubscribeFromBlockZero *bool `yaml:"stripSubscribeFromBlockZero,omitempty" json:"stripSubscribeFromBlockZero,omitempty"` + // DynamicBlockTimeDebounceMultiplier scales the EMA-estimated block time to derive // the debounce interval for block polling. A value of 0.7 means debounce = 70% of // the estimated block time, preferring fresher data at the cost of slightly more diff --git a/common/config_test.go b/common/config_test.go index 5a0ff3f1b..9963e5f5f 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -49,6 +49,30 @@ logLevel: DEBUG } } +// Configs that still set the ignored indexer.canonicalChainDepth must load +// under strict decoding. +func TestLoadConfig_DeprecatedIndexerCanonicalChainDepthAccepted(t *testing.T) { + fs := afero.NewMemMapFs() + cfg, err := afero.TempFile(fs, "", "erpc.yaml") + if err != nil { + t.Fatal(err) + } + cfg.WriteString(` +logLevel: error +indexer: + canonicalChainDepth: 256 + dedupWindowSize: 1024 +`) + + loaded, err := LoadConfig(fs, cfg.Name(), &DefaultOptions{}) + if err != nil { + t.Fatalf("config setting the deprecated field must still load: %v", err) + } + if loaded.Indexer == nil || loaded.Indexer.DedupWindowSize != 1024 { + t.Fatalf("indexer block must still parse, got %+v", loaded.Indexer) + } +} + // TestLoadConfig_LegacyFieldsAccepted pins the back-compat contract: // a prod-shape YAML carrying legacy `routing.scoreMultipliers` on // upstreams + `scoreMetricsWindowSize` at the project level loads @@ -659,6 +683,22 @@ failsafe: assert.Equal(t, 2*time.Second, config.Failsafe[0].Timeout.Duration.Resolve(nil)) assert.Equal(t, 3, config.Failsafe[0].Retry.MaxAttempts) }) + + t.Run("old single object format keeps failover", func(t *testing.T) { + yamlData := ` +architecture: evm +failsafe: + timeout: + duration: 2s +failover: + onDefaultsExhausted: true +` + var config NetworkConfig + err := yaml.Unmarshal([]byte(yamlData), &config) + assert.NoError(t, err) + assert.Len(t, config.Failsafe, 1) + assert.True(t, config.Failover.Enabled()) + }) } func TestNetworkDefaultsFailsafeBackwardCompatibility(t *testing.T) { @@ -697,6 +737,22 @@ failsafe: assert.Equal(t, "*", defaults.Failsafe[0].MatchMethod) // Should default to "*" assert.Equal(t, 2*time.Second, defaults.Failsafe[0].Timeout.Duration.Resolve(nil)) }) + + t.Run("old single object format keeps failover", func(t *testing.T) { + yamlData := ` +rateLimitBudget: "test" +failsafe: + timeout: + duration: 2s +failover: + onDefaultsExhausted: true +` + var defaults NetworkDefaults + err := yaml.Unmarshal([]byte(yamlData), &defaults) + assert.NoError(t, err) + assert.Len(t, defaults.Failsafe, 1) + assert.True(t, defaults.Failover.Enabled()) + }) } func TestUpstreamConfigFailsafeBackwardCompatibility(t *testing.T) { @@ -1212,6 +1268,35 @@ projects: }) } +func TestNetworkConfig_SetDefaults_FailoverInheritsFromDefaults(t *testing.T) { + enabled := true + defaults := &NetworkDefaults{ + Failover: &FailoverConfig{OnDefaultsExhausted: &enabled}, + } + n := &NetworkConfig{Architecture: ArchitectureEvm, Evm: &EvmNetworkConfig{ChainId: 1}} + err := n.SetDefaults(nil, defaults) + assert.NoError(t, err) + assert.NotNil(t, n.Failover) + assert.True(t, n.Failover.Enabled()) +} + +func TestFailoverConfig_Enabled(t *testing.T) { + var nilCfg *FailoverConfig + assert.False(t, nilCfg.Enabled()) + assert.False(t, (&FailoverConfig{}).Enabled()) + disabled, enabled := false, true + assert.False(t, (&FailoverConfig{OnDefaultsExhausted: &disabled}).Enabled()) + assert.True(t, (&FailoverConfig{OnDefaultsExhausted: &enabled}).Enabled()) +} + +func TestNetworkConfig_SetDefaults_StripSubscribeFromBlockZeroInheritsFromDefaults(t *testing.T) { + strip := true + defaults := &NetworkDefaults{Evm: &EvmNetworkConfig{StripSubscribeFromBlockZero: &strip}} + n := &NetworkConfig{Architecture: ArchitectureEvm, Evm: &EvmNetworkConfig{ChainId: 1}} + assert.NoError(t, n.SetDefaults(nil, defaults)) + assert.Equal(t, &strip, n.Evm.StripSubscribeFromBlockZero) +} + func TestUpstreamConfig_ValidateRateLimitCountMode(t *testing.T) { cfg := &Config{} base := func(mode RateLimitCountMode) *UpstreamConfig { diff --git a/common/defaults.go b/common/defaults.go index 74e4829cf..42269995b 100644 --- a/common/defaults.go +++ b/common/defaults.go @@ -881,6 +881,33 @@ func (s *ServerConfig) SetDefaults() error { s.CostHeaders = util.BoolPtr(false) } + if s.WebSocket == nil { + s.WebSocket = &WebSocketServerConfig{} + } + if s.WebSocket.ReadBufferSize == 0 { + s.WebSocket.ReadBufferSize = 4096 + } + if s.WebSocket.WriteBufferSize == 0 { + s.WebSocket.WriteBufferSize = 4096 + } + if s.WebSocket.MaxMessageSize == 0 { + s.WebSocket.MaxMessageSize = 1 * 1024 * 1024 // 1MB + } + if s.WebSocket.PingInterval == nil { + d := Duration(30 * time.Second) + s.WebSocket.PingInterval = &d + } + if s.WebSocket.MaxSubscriptionsPerConnection == 0 { + s.WebSocket.MaxSubscriptionsPerConnection = 100 + } + if s.WebSocket.MaxConcurrentRequestsPerConnection == 0 { + // Same as the HTTP/2 per-connection stream limit on this listener. + s.WebSocket.MaxConcurrentRequestsPerConnection = 250 + } + if s.WebSocket.SubscriptionBufferSize == 0 { + // Absorbs a block's worth of matching logs for a briefly slow client. + s.WebSocket.SubscriptionBufferSize = 256 + } // Safe defaults for client IP resolution if len(s.TrustedIPForwarders) == 0 { // Only loopback by default; do not trust private subnets unless explicitly configured @@ -1397,6 +1424,8 @@ func (p *ProjectConfig) SetDefaults(opts *DefaultOptions) error { func convertUpstreamToProvider(upstream *UpstreamConfig) (*ProviderConfig, error) { if strings.HasPrefix(upstream.Endpoint, "http://") || strings.HasPrefix(upstream.Endpoint, "https://") || + strings.HasPrefix(upstream.Endpoint, "ws://") || + strings.HasPrefix(upstream.Endpoint, "wss://") || strings.HasPrefix(upstream.Endpoint, "grpc://") || strings.HasPrefix(upstream.Endpoint, "grpc+bds://") { return nil, nil @@ -2191,6 +2220,14 @@ func (n *NetworkConfig) SetDefaults(upstreams []*UpstreamConfig, defaults *Netwo v := *defaults.Multiplexing n.Multiplexing = &v } + if n.Failover == nil && defaults.Failover != nil { + cp := *defaults.Failover + if defaults.Failover.OnDefaultsExhausted != nil { + v := *defaults.Failover.OnDefaultsExhausted + cp.OnDefaultsExhausted = &v + } + n.Failover = &cp + } if n.Evm != nil && defaults.Evm != nil { if n.Evm.Integrity == nil && defaults.Evm.Integrity != nil { n.Evm.Integrity = &EvmIntegrityConfig{} @@ -2242,6 +2279,9 @@ func (n *NetworkConfig) SetDefaults(upstreams []*UpstreamConfig, defaults *Netwo if n.Evm.EmptyResultConfidence == 0 && defaults.Evm.EmptyResultConfidence != 0 { n.Evm.EmptyResultConfidence = defaults.Evm.EmptyResultConfidence } + if n.Evm.StripSubscribeFromBlockZero == nil && defaults.Evm.StripSubscribeFromBlockZero != nil { + n.Evm.StripSubscribeFromBlockZero = defaults.Evm.StripSubscribeFromBlockZero + } } else if n.Evm == nil && defaults.Evm != nil && n.Svm == nil && n.Architecture != ArchitectureSvm { // Copy EVM defaults only onto networks that are (or can become) EVM. // Without the SVM guard, a mixed project with networkDefaults.evm @@ -2304,7 +2344,7 @@ func (n *NetworkConfig) SetDefaults(upstreams []*UpstreamConfig, defaults *Netwo if len(upstreams) > 0 { anyUpstreamInFallbackTier := slices.ContainsFunc(upstreams, func(u *UpstreamConfig) bool { - return u.HasTag("tier:fallback") + return u.HasTag(TagTierFallback) }) if anyUpstreamInFallbackTier && n.SelectionPolicy == nil { defCfg := NewDefaultNetworkConfig(upstreams) diff --git a/common/errors.go b/common/errors.go index 23c9c181d..ed62b26cc 100644 --- a/common/errors.go +++ b/common/errors.go @@ -3068,3 +3068,67 @@ var NewErrEndpointNonceException = func(cause error, reason NonceExceptionReason func (e *ErrEndpointNonceException) ErrorStatusCode() int { return http.StatusOK } + +// +// WebSocket / Subscription errors +// + +type ErrSubscriptionNotFound struct{ BaseError } + +const ErrCodeSubscriptionNotFound ErrorCode = "ErrSubscriptionNotFound" + +var NewErrSubscriptionNotFound = func(subId string) error { + return &ErrSubscriptionNotFound{ + BaseError{ + Code: ErrCodeSubscriptionNotFound, + Message: "subscription not found", + Details: map[string]interface{}{ + "subscriptionId": subId, + }, + }, + } +} + +func (e *ErrSubscriptionNotFound) ErrorStatusCode() int { + return http.StatusNotFound +} + +type ErrNoWsUpstreamAvailable struct{ BaseError } + +const ErrCodeNoWsUpstreamAvailable ErrorCode = "ErrNoWsUpstreamAvailable" + +var NewErrNoWsUpstreamAvailable = func(networkId string) error { + return &ErrNoWsUpstreamAvailable{ + BaseError{ + Code: ErrCodeNoWsUpstreamAvailable, + Message: fmt.Sprintf("eth_subscribe requires a WebSocket-capable upstream, none configured for network %s", networkId), + Details: map[string]interface{}{ + "networkId": networkId, + }, + }, + } +} + +func (e *ErrNoWsUpstreamAvailable) ErrorStatusCode() int { + return http.StatusBadRequest +} + +type ErrSubscriptionLimitExceeded struct{ BaseError } + +const ErrCodeSubscriptionLimitExceeded ErrorCode = "ErrSubscriptionLimitExceeded" + +var NewErrSubscriptionLimitExceeded = func(max int) error { + return &ErrSubscriptionLimitExceeded{ + BaseError{ + Code: ErrCodeSubscriptionLimitExceeded, + Message: "maximum subscriptions per connection exceeded", + Details: map[string]interface{}{ + "maxPerConnection": max, + }, + }, + } +} + +func (e *ErrSubscriptionLimitExceeded) ErrorStatusCode() int { + return http.StatusTooManyRequests +} diff --git a/common/request.go b/common/request.go index 3a2943fe9..da8e5d5b9 100644 --- a/common/request.go +++ b/common/request.go @@ -240,6 +240,10 @@ type NormalizedRequest struct { upstreamList []Upstream // Available upstreams for this request ConsumedUpstreams *sync.Map // Tracks upstreams that provided valid responses + // escalatedToFallbacks makes the per-request fallback escape happen at + // most once across retries and hedges. + escalatedToFallbacks atomic.Bool + lastValidResponse atomic.Pointer[NormalizedResponse] integrityCaught atomic.Bool // an integrity check rejected a response during this request integrityRejectedCheck atomic.Value // id of the last check that rejected (the "why") @@ -1286,6 +1290,15 @@ func (r *NormalizedRequest) Upstreams() []Upstream { return out } +// MarkEscalatedToFallbacks reports true only for the first caller, so a +// request escalates to the fallback tier at most once. +func (r *NormalizedRequest) MarkEscalatedToFallbacks() bool { + if r == nil { + return false + } + return r.escalatedToFallbacks.CompareAndSwap(false, true) +} + // UserId returns the user ID from the user object, or "n/a" if not available func (r *NormalizedRequest) UserId() string { if r == nil { diff --git a/common/validation.go b/common/validation.go index 84d3b5ca5..ff71eee50 100644 --- a/common/validation.go +++ b/common/validation.go @@ -65,6 +65,20 @@ func (c *Config) Validate() error { } } } + if c.Indexer != nil { + if err := c.Indexer.Validate(); err != nil { + return err + } + } + return nil +} + +// Validate rejects nonsensical values; zeros pass through and resolve to +// internal defaults inside the indexer. +func (i *IndexerConfig) Validate() error { + if i.DedupWindowSize < 0 { + return fmt.Errorf("indexer.dedupWindowSize must be >= 0 (0 uses the default)") + } return nil } @@ -92,6 +106,23 @@ func (s *ServerConfig) Validate() error { if s.MaxTimeout == nil || *s.MaxTimeout == 0 { return fmt.Errorf("server.maxTimeout is required") } + if ws := s.WebSocket; ws != nil { + if ws.PingInterval == nil || *ws.PingInterval <= 0 { + return fmt.Errorf("server.webSocket.pingInterval must be greater than 0") + } + if ws.ReadBufferSize < 0 || ws.WriteBufferSize < 0 || ws.MaxMessageSize < 0 { + return fmt.Errorf("server.webSocket buffer and message sizes must not be negative") + } + if ws.MaxSubscriptionsPerConnection < 1 { + return fmt.Errorf("server.webSocket.maxSubscriptionsPerConnection must be at least 1") + } + if ws.MaxConcurrentRequestsPerConnection < 1 { + return fmt.Errorf("server.webSocket.maxConcurrentRequestsPerConnection must be at least 1") + } + if ws.SubscriptionBufferSize < 1 { + return fmt.Errorf("server.webSocket.subscriptionBufferSize must be at least 1") + } + } // Validate trusted IP forwarders if provided (IPs or CIDRs). Support legacy + new field for _, entry := range s.TrustedIPForwarders { diff --git a/common/validation_websocket_test.go b/common/validation_websocket_test.go new file mode 100644 index 000000000..18e4aba47 --- /dev/null +++ b/common/validation_websocket_test.go @@ -0,0 +1,38 @@ +package common + +import ( + "testing" + + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServerConfig_Validate_WebSocket(t *testing.T) { + defaulted := func() *ServerConfig { + s := &ServerConfig{ListenV4: util.BoolPtr(false), ListenV6: util.BoolPtr(false)} + require.NoError(t, s.SetDefaults()) + return s + } + + t.Run("defaults are valid", func(t *testing.T) { + require.NoError(t, defaulted().Validate()) + }) + + cases := map[string]func(ws *WebSocketServerConfig){ + "pingInterval": func(ws *WebSocketServerConfig) { d := Duration(0); ws.PingInterval = &d }, + "sizes": func(ws *WebSocketServerConfig) { ws.MaxMessageSize = -1 }, + "maxSubscriptionsPerConnection": func(ws *WebSocketServerConfig) { ws.MaxSubscriptionsPerConnection = -1 }, + "maxConcurrentRequestsPerConnection": func(ws *WebSocketServerConfig) { ws.MaxConcurrentRequestsPerConnection = -1 }, + "subscriptionBufferSize": func(ws *WebSocketServerConfig) { ws.SubscriptionBufferSize = -1 }, + } + for field, mutate := range cases { + t.Run("invalid "+field+" rejected", func(t *testing.T) { + s := defaulted() + mutate(s.WebSocket) + err := s.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), field) + }) + } +} diff --git a/erpc/config_analyzer.go b/erpc/config_analyzer.go index 9e1b9d36e..8c4d50441 100644 --- a/erpc/config_analyzer.go +++ b/erpc/config_analyzer.go @@ -257,6 +257,15 @@ func GenerateValidationReport(ctx context.Context, cfg *common.Config) *Validati } } } + + // Warnings: failover enabled but no fallback-tier upstream to escalate to. + failoverEnabled := p.NetworkDefaults != nil && p.NetworkDefaults.Failover.Enabled() + for _, nw := range p.Networks { + failoverEnabled = failoverEnabled || nw.Failover.Enabled() + } + if failoverEnabled && !projectHasFallbackTier(p) { + report.Warnings = append(report.Warnings, fmt.Sprintf("project=%s failover.onDefaultsExhausted is enabled but no upstream is tagged '%s', so requests never escalate", p.Id, common.TagTierFallback)) + } } // Admin auth budgets @@ -280,6 +289,11 @@ func GenerateValidationReport(ctx context.Context, cfg *common.Config) *Validati // Upstream runtime checks (chain id + block hash comparisons). Use a silent logger and short timeout per upstream silent := zerolog.New(io.Discard) + // Transient upstreams below spawn client goroutines bound to their ctx; + // cancel it on return so they don't outlive validation. + valCtx, cancelVal := context.WithCancel(ctx) + defer cancelVal() + // Histogram buckets (validate config value) if err := telemetry.SetHistogramBuckets(cfg.Metrics.HistogramBuckets); err != nil { report.Errors = append(report.Errors, fmt.Sprintf("invalid metrics histogramBuckets: %v", err)) @@ -327,7 +341,7 @@ func GenerateValidationReport(ctx context.Context, cfg *common.Config) *Validati } clReg := clients.NewClientRegistry(&silent, project.Id, prxPool, upstream.NewCompositeJsonRpcErrorExtractor()) vndReg := thirdparty.NewVendorsRegistry() - rlr, err := upstream.NewRateLimitersRegistry(ctx, cfg.RateLimiters, &silent) + rlr, err := upstream.NewRateLimitersRegistry(valCtx, cfg.RateLimiters, &silent) if err != nil { appendErr(fmt.Sprintf("project=%s failed to create rate limiters registry: %v", project.Id, err)) continue @@ -346,7 +360,7 @@ func GenerateValidationReport(ctx context.Context, cfg *common.Config) *Validati } // Create upstream - ups, err := upstream.NewUpstream(ctx, prj, uc, clReg, rlr, vndReg, &silent, mt, nil) + ups, err := upstream.NewUpstream(valCtx, prj, uc, clReg, rlr, vndReg, &silent, mt, nil) if err != nil { appendErr(fmt.Sprintf("project=%s upstream=%s failed to create upstream: %v", prj, uc.Id, err)) return @@ -952,6 +966,24 @@ func calculateConfigStats(cfg *common.Config) ConfigStats { return stats } +// projectHasFallbackTier reports whether any upstream the project defines, +// directly or via provider overrides, is tagged as fallback tier. +func projectHasFallbackTier(p *common.ProjectConfig) bool { + for _, u := range p.Upstreams { + if u.HasTag(common.TagTierFallback) { + return true + } + } + for _, pr := range p.Providers { + for _, o := range pr.Overrides { + if o != nil && o.HasTag(common.TagTierFallback) { + return true + } + } + } + return false +} + func printConfigStats(logger zerolog.Logger, stats ConfigStats) { logger.Info().Msg("Configuration Statistics:") @@ -988,6 +1020,10 @@ func printConfigStats(logger zerolog.Logger, stats ConfigStats) { } func validateUpstreamEndpoints(ctx context.Context, cfg *common.Config, logger zerolog.Logger) error { + // Transient upstreams below spawn client goroutines bound to their ctx; + // cancel it on return so they don't outlive validation. + valCtx, cancelVal := context.WithCancel(ctx) + defer cancelVal() err := telemetry.SetHistogramBuckets( cfg.Metrics.HistogramBuckets, ) @@ -1014,7 +1050,7 @@ func validateUpstreamEndpoints(ctx context.Context, cfg *common.Config, logger z ) vndReg := thirdparty.NewVendorsRegistry() rlr, err := upstream.NewRateLimitersRegistry( - ctx, + valCtx, cfg.RateLimiters, &logger, ) @@ -1050,7 +1086,7 @@ func validateUpstreamEndpoints(ctx context.Context, cfg *common.Config, logger z continue } ups, err := upstream.NewUpstream( - ctx, + valCtx, project.Id, upsCfg, clReg, @@ -1063,7 +1099,7 @@ func validateUpstreamEndpoints(ctx context.Context, cfg *common.Config, logger z if err != nil { return fmt.Errorf("failed to create upstream for project: \"%s\" and upstream id: \"%s\": %w", project.Id, upsCfg.Id, err) } - chainStr, err := ups.EvmGetChainId(ctx) + chainStr, err := ups.EvmGetChainId(valCtx) if err != nil { return fmt.Errorf("failed to get chain id for project: \"%s\" and upstream id: \"%s\": %w", project.Id, upsCfg.Id, err) } diff --git a/erpc/config_analyzer_test.go b/erpc/config_analyzer_test.go new file mode 100644 index 000000000..39eb91fac --- /dev/null +++ b/erpc/config_analyzer_test.go @@ -0,0 +1,42 @@ +package erpc + +import ( + "context" + "strings" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/assert" +) + +func TestGenerateValidationReport_WarnsOnFailoverWithoutFallbackTier(t *testing.T) { + hasWarning := func(r *ValidationReport) bool { + for _, w := range r.Warnings { + if strings.Contains(w, "failover.onDefaultsExhausted") { + return true + } + } + return false + } + newCfg := func(tags []string) *common.Config { + return &common.Config{ + Metrics: &common.MetricsConfig{}, + Projects: []*common.ProjectConfig{{ + Id: "main", + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 1}, + Failover: &common.FailoverConfig{OnDefaultsExhausted: util.BoolPtr(true)}, + }}, + Providers: []*common.ProviderConfig{{ + Id: "p", + Overrides: map[string]*common.UpstreamConfig{"*": {Tags: tags}}, + }}, + }}, + } + } + + assert.True(t, hasWarning(GenerateValidationReport(context.Background(), newCfg(nil)))) + assert.False(t, hasWarning(GenerateValidationReport(context.Background(), newCfg([]string{common.TagTierFallback})))) +} diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 34d50d7bc..033c525e5 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -117,6 +117,18 @@ var ( Help: "Dynamically computed block time per network in milliseconds.", }, []string{"project", "network"}) + MetricUpstreamWebsocketConnected = DefineGauge(prometheus.GaugeOpts{ + Namespace: "erpc", + Name: "upstream_websocket_connected", + Help: "Whether the upstream WebSocket connection is currently established (1) or down/wedged (0).", + }, []string{"project", "vendor", "network", "upstream"}) + + MetricWebsocketSubscriptionNotificationsDroppedTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "websocket_subscription_notifications_dropped_total", + Help: "Subscription notifications dropped because a client's per-subscription buffer was full. For logs the client connection is closed with 1013.", + }, []string{"project", "network", "kind"}) + // MetricNetworkServedTipBlockNumber is the block number the network actually // advertises/serves as the tip for a block tag (axis=latest|finalized): the // freshest block a strict MAJORITY of eligible upstreams already have (see @@ -738,6 +750,15 @@ var ( Help: "Total circuit-breaker state transitions per upstream and direction (closed_to_open/half_open_to_open/half_open_to_closed/open_to_half_open).", }, []string{"project", "upstream", "transition"}) + // MetricNetworkFallbackEscapeTotal counts requests that escalated to the + // fallback tier after exhausting the routed upstreams. Expected to be zero + // in steady state. + MetricNetworkFallbackEscapeTotal = DefineCounter(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "network_fallback_escape_total", + Help: "Total number of times the per-request fallback escape hatch fired because the primary upstream set was exhausted with retryable errors.", + }, []string{"project", "network", "category"}) + // MetricCacheExecutorAttempt counts every attempt a cache-connector // failsafe executor governs, keyed by the executor's identity (the // matchMethod / matchFinality it was configured with) and the outcome diff --git a/typescript/config/src/generated.ts b/typescript/config/src/generated.ts index ce0d460b0..862d69821 100644 --- a/typescript/config/src/generated.ts +++ b/typescript/config/src/generated.ts @@ -222,9 +222,25 @@ export interface Config { projects?: (ProjectConfig | undefined)[]; rateLimiters?: RateLimiterConfig; metrics?: MetricsConfig; + indexer?: IndexerConfig; proxyPools?: (ProxyPoolConfig | undefined)[]; tracing?: TracingConfig; } +/** + * IndexerConfig tunes the event-stream indexer behind eth_subscribe fan-out. + */ +export interface IndexerConfig { + /** + * Deprecated: ignored; the indexer no longer tracks a canonical chain. + * Kept so configs that still set it load under strict decoding. + */ + canonicalChainDepth?: number /* int */; + /** + * DedupWindowSize is the per-filter seen-set capacity for log and + * pending-tx fan-out across upstreams. 0 uses the default (8192). + */ + dedupWindowSize?: number /* int */; +} export interface ServerConfig { listenV4?: boolean; httpHostV4?: string; @@ -253,6 +269,7 @@ export interface ServerConfig { trustedIPForwarders?: string[]; trustedIPHeaders?: string[]; responseHeaders?: { [key: string]: string}; + webSocket?: WebSocketServerConfig; /** * ExecutionHeaders controls the per-request diagnostic headers * (X-ERPC-Attempts, X-ERPC-Upstreams-Tried, etc.) that expose how @@ -270,6 +287,26 @@ export interface ServerConfig { */ costHeaders?: boolean; } +export interface WebSocketServerConfig { + readBufferSize?: number /* int */; + writeBufferSize?: number /* int */; + maxMessageSize?: number /* int64 */; + pingInterval?: Duration; + maxSubscriptionsPerConnection?: number /* int */; + /** + * MaxConcurrentRequestsPerConnection bounds the messages a connection has + * in flight; further frames wait (backpressure) instead of being rejected. + * A batch is one message; its requests run concurrently, as over HTTP. + */ + maxConcurrentRequestsPerConnection?: number /* int */; + /** + * SubscriptionBufferSize is the number of notifications queued per + * subscription for a client that is behind. On overflow newHeads and + * newPendingTransactions drop their oldest entry; logs close the + * connection instead (1013), so the client knows it missed data. + */ + subscriptionBufferSize?: number /* int */; +} /** * ExecutionHeadersMode controls how much per-request execution detail is * exposed in HTTP response headers. @@ -716,6 +753,19 @@ export interface NetworkDefaults { svm?: TsSvmNetworkConfigForDefaults; multiplexing?: boolean; cacheKeySuffix?: string; + failover?: FailoverConfig; +} +/** + * FailoverConfig controls per-request escalation to fallback-tier upstreams, + * independent of the selection policy. + */ +export interface FailoverConfig { + /** + * OnDefaultsExhausted tries upstreams not tagged `tier:fallback` first and + * advances to the fallback tier within the same request only once every + * other upstream failed with an error retryable toward the network. + */ + onDefaultsExhausted?: boolean; } export interface CORSConfig { allowedOrigins: string[]; @@ -735,6 +785,11 @@ export interface ProviderConfig { upstreamIdTemplate?: string; overrides?: { [key: string]: UpstreamConfig | undefined}; } +/** + * TagTierFallback marks an upstream as part of the fallback tier, used only + * when the other upstreams are unavailable. + */ +export const TagTierFallback = "tier:fallback"; /** * RateLimitCountMode selects the accounting unit an upstream's rate-limit * budget charges per call. @@ -1371,6 +1426,7 @@ export interface NetworkConfig { methods?: MethodsConfig; multiplexing?: boolean; staticResponses?: (StaticResponseConfig | undefined)[]; + failover?: FailoverConfig; /** * Integrity overrides the project-wide data-integrity configuration for this * network. Merges over the project block (network wins). @@ -1603,6 +1659,13 @@ export interface EvmNetworkConfig { * Default includes common point-lookup methods like eth_getBlockByNumber, eth_getTransactionByHash, etc. */ markEmptyAsErrorMethods?: string[]; + /** + * StripSubscribeFromBlockZero removes a zero `fromBlock` from eth_subscribe + * logs filters before forwarding upstream. fromBlock has no meaning for a + * live subscription, and backends that prune history reject it. + * DEFAULT: false. + */ + stripSubscribeFromBlockZero?: boolean; /** * DynamicBlockTimeDebounceMultiplier scales the EMA-estimated block time to derive * the debounce interval for block polling. A value of 0.7 means debounce = 70% of From 8804351351062fe678e32099141d27d11eb130f5 Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 13:27:51 +0100 Subject: [PATCH 03/21] feat(clients): WebSocket JSON-RPC upstream client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ws:// and wss:// endpoints get a persistent client instead of the "not implemented" stub. It serves regular requests and upstream subscriptions: - one read loop owns the connection lifecycle; re-dials wait a jittered exponential backoff (1s-30s) that resets only after a connection outlives the liveness window; ping/pong plus a read deadline detect half-open connections; - requests fail fast with a retryable error while reconnecting and errors are normalised exactly like the HTTP client's; - subscription handlers are registered before the next frame is read and are scoped to the connection that created them; eth_subscribe and eth_unsubscribe are only sent on behalf of a subscription owner. Upstream.IsDown checks the breaker of the policy serving a method, and the registry exposes the WebSocket and fallback-escape upstream sets. A response the read loop already claimed reaches its caller even if the caller's context expired meanwhile, so a subscribe that completed is never left without an owner; a notification for a subscription no handler owns triggers eth_unsubscribe, which also retries a cancel that failed. A panic in a subscription handler or callback is recovered per frame. jsonRpc.proxyPool is refused for ws/wss upstreams, whose dialer has no proxy support. Co-authored-by: snowkide Co-authored-by: João Gomes Co-Authored-By: Claude Opus 5.5 --- clients/registry.go | 48 +- clients/registry_test.go | 49 ++ clients/ws_json_rpc_client.go | 771 ++++++++++++++++++++++ clients/ws_json_rpc_client_errors_test.go | 76 +++ clients/ws_json_rpc_client_test.go | 698 ++++++++++++++++++++ go.mod | 1 + go.sum | 2 + upstream/registry.go | 42 ++ upstream/registry_fallback_escape_test.go | 194 ++++++ upstream/upstream.go | 17 +- upstream/upstream_isdown_test.go | 42 ++ 11 files changed, 1932 insertions(+), 8 deletions(-) create mode 100644 clients/registry_test.go create mode 100644 clients/ws_json_rpc_client.go create mode 100644 clients/ws_json_rpc_client_errors_test.go create mode 100644 clients/ws_json_rpc_client_test.go create mode 100644 upstream/registry_fallback_escape_test.go create mode 100644 upstream/upstream_isdown_test.go diff --git a/clients/registry.go b/clients/registry.go index 558876499..aac79bc8f 100644 --- a/clients/registry.go +++ b/clients/registry.go @@ -15,6 +15,7 @@ type ClientType string const ( ClientTypeHttpJsonRpc ClientType = "HttpJsonRpc" ClientTypeGrpcBds ClientType = "GrpcBds" + ClientTypeWsJsonRpc ClientType = "WsJsonRpc" ) type ClientInterface interface { @@ -26,10 +27,19 @@ type Client struct { Upstream common.Upstream } +// clientCreation lets concurrent callers that miss the cache share one client +// instead of each spawning one. Failed creations are not memoised. +type clientCreation struct { + once sync.Once + client ClientInterface + err error +} + type ClientRegistry struct { logger *zerolog.Logger projectId string clients sync.Map + clientCreations sync.Map // upstream key -> *clientCreation proxyPoolRegistry *ProxyPoolRegistry evmExtractor common.JsonRpcErrorExtractor } @@ -53,7 +63,6 @@ func (manager *ClientRegistry) GetOrCreateClient(appCtx context.Context, ups com } func (manager *ClientRegistry) CreateClient(appCtx context.Context, ups common.Upstream) (ClientInterface, error) { - var once sync.Once var newClient ClientInterface var clientErr error @@ -76,10 +85,14 @@ func (manager *ClientRegistry) CreateClient(appCtx context.Context, ups common.U } } + upstreamKey := common.UniqueUpstreamKey(ups) + cv, _ := manager.clientCreations.LoadOrStore(upstreamKey, &clientCreation{}) + creation := cv.(*clientCreation) + if err != nil { clientErr = fmt.Errorf("failed to parse URL for upstream: %v", cfg.Id) } else { - once.Do(func() { + creation.once.Do(func() { lg := manager.logger.With().Str("upstreamId", cfg.Id).Logger() switch cfg.Type { case common.UpstreamTypeEvm: @@ -95,10 +108,27 @@ func (manager *ClientRegistry) CreateClient(appCtx context.Context, ups common.U manager.evmExtractor, ) if err != nil { - clientErr = fmt.Errorf("failed to create HTTP client for upstream: %v", cfg.Id) + clientErr = fmt.Errorf("failed to create HTTP client for upstream: %v: %w", cfg.Id, err) } } else if parsedUrl.Scheme == "ws" || parsedUrl.Scheme == "wss" { - clientErr = fmt.Errorf("websocket client not implemented yet") + // The WebSocket dialer does not go through a proxy pool; + // refuse rather than silently dial direct. + if proxyPool != nil { + clientErr = fmt.Errorf("jsonRpc.proxyPool is not supported for WebSocket upstream %v", cfg.Id) + } else { + newClient, err = NewWsJsonRpcClient( + appCtx, + &lg, + manager.projectId, + ups, + parsedUrl, + cfg.JsonRpc, + manager.evmExtractor, + ) + if err != nil { + clientErr = fmt.Errorf("failed to create WebSocket client for upstream %v: %w", cfg.Id, err) + } + } } else if parsedUrl.Scheme == "grpc" || parsedUrl.Scheme == "grpc+bds" { grpcPoolSize, grpcHealthCheckService := 0, "" if cfg.Grpc != nil { @@ -115,7 +145,7 @@ func (manager *ClientRegistry) CreateClient(appCtx context.Context, ups common.U grpcHealthCheckService, ) if err != nil { - clientErr = fmt.Errorf("failed to create gRPC BDS client for upstream: %v", cfg.Id) + clientErr = fmt.Errorf("failed to create gRPC BDS client for upstream: %v: %w", cfg.Id, err) } } else { clientErr = fmt.Errorf("unsupported endpoint scheme: %v for upstream: %v", parsedUrl.Scheme, cfg.Id) @@ -137,7 +167,7 @@ func (manager *ClientRegistry) CreateClient(appCtx context.Context, ups common.U manager.evmExtractor, ) if err != nil { - clientErr = fmt.Errorf("failed to create HTTP client for upstream: %v", cfg.Id) + clientErr = fmt.Errorf("failed to create HTTP client for upstream: %v: %w", cfg.Id, err) } } else { clientErr = fmt.Errorf("unsupported endpoint scheme for svm upstream %v: %v (only http/https supported)", cfg.Id, parsedUrl.Scheme) @@ -147,10 +177,14 @@ func (manager *ClientRegistry) CreateClient(appCtx context.Context, ups common.U clientErr = fmt.Errorf("unsupported upstream type: %v for upstream: %v", cfg.Type, cfg.Id) } + creation.client, creation.err = newClient, clientErr if clientErr == nil { - manager.clients.Store(common.UniqueUpstreamKey(ups), newClient) + manager.clients.Store(upstreamKey, newClient) + } else { + manager.clientCreations.CompareAndDelete(upstreamKey, creation) } }) + newClient, clientErr = creation.client, creation.err } return newClient, clientErr diff --git a/clients/registry_test.go b/clients/registry_test.go new file mode 100644 index 000000000..6f4ef1d79 --- /dev/null +++ b/clients/registry_test.go @@ -0,0 +1,49 @@ +package clients + +import ( + "context" + "testing" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +func TestClientRegistryRetriesFailedCreation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := zerolog.Nop() + registry := NewClientRegistry(&logger, "test-project", nil, nil) + + up := common.NewFakeUpstream("test-upstream") + up.Config().Endpoint = "http://rpc1.localhost" + + _, err := registry.GetOrCreateClient(ctx, up) + require.Error(t, err, "creation should fail while the upstream type is unset") + + up.Config().Type = common.UpstreamTypeEvm + client, err := registry.GetOrCreateClient(ctx, up) + require.NoError(t, err, "a failed creation must not be memoised") + require.NotNil(t, client) +} + +// The WebSocket dialer does not use a proxy pool, so a WebSocket upstream +// that names one must fail instead of silently dialling direct. +func TestClientRegistryRejectsProxyPoolForWebSocket(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := zerolog.Nop() + proxies, err := NewProxyPoolRegistry([]*common.ProxyPoolConfig{ + {ID: "pool1", Urls: []string{"http://myproxy1:8080"}}, + }, &logger) + require.NoError(t, err) + registry := NewClientRegistry(&logger, "test-project", proxies, nil) + + up := common.NewFakeUpstream("ws-upstream") + up.Config().Type = common.UpstreamTypeEvm + up.Config().Endpoint = "wss://rpc1.localhost" + up.Config().JsonRpc = &common.JsonRpcUpstreamConfig{ProxyPool: "pool1"} + + _, err = registry.GetOrCreateClient(ctx, up) + require.ErrorContains(t, err, "proxyPool is not supported for WebSocket") +} diff --git a/clients/ws_json_rpc_client.go b/clients/ws_json_rpc_client.go new file mode 100644 index 000000000..d2f0de374 --- /dev/null +++ b/clients/ws_json_rpc_client.go @@ -0,0 +1,771 @@ +package clients + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "net" + "net/http" + "net/url" + "runtime/debug" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/gorilla/websocket" + "github.com/rs/zerolog" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const ( + wsWriteWait = 10 * time.Second + wsReconnectMin = 1 * time.Second + wsReconnectMax = 30 * time.Second + wsReconnectFactor = 2.0 +) + +// Liveness windows: a connection with no pong or data frame for wsPongWait +// is torn down and re-dialed, since a half-open connection would otherwise +// block reads forever while pings keep "succeeding" into a buffer. Two +// pings fit in the window. Vars so tests can shorten them. +var ( + wsPingInterval = 30 * time.Second + wsPongWait = 75 * time.Second +) + +// WsJsonRpcClient implements ClientInterface for WebSocket-based JSON-RPC upstream connections. +type WsJsonRpcClient struct { + Url *url.URL + headers http.Header + + projectId string + upstream common.Upstream + appCtx context.Context + logger *zerolog.Logger + + pingInterval time.Duration + pongWait time.Duration + + // Connection state. Only readLoop replaces or tears down conn (besides + // the initial dial and shutdown); the dial itself runs without connMu so + // requests never wait on a handshake. + connMu sync.Mutex + conn *websocket.Conn + connectedAt time.Time + // epoch identifies the current connection (0 while disconnected); + // written under connMu, epochs counts connections ever installed. + epoch atomic.Uint64 + epochs uint64 + + // Write synchronization (gorilla/websocket requires synchronized writes) + writeMu sync.Mutex + + // Pending request tracking: wire id -> request. Whoever deletes an + // entry owns it, so duplicate responses are dropped. + pendingMu sync.Mutex + pending map[string]*wsPending + + // Subscription notification callbacks: upstreamSubID -> handler, for + // the current connection only (ids are connection-scoped). + subHandlersMu sync.RWMutex + subHandlers map[string]func(params []byte) + + // Disconnect/reconnect callbacks, keyed by caller-supplied id so a + // caller registering again replaces its callback. + onDisconnectMu sync.RWMutex + onDisconnectCbs map[string]func() + + onReconnectMu sync.RWMutex + onReconnectCbs map[string]func() + + errorExtractor common.JsonRpcErrorExtractor + + // metricLabels are the labels the connectivity gauge was last set under. + metricMu sync.Mutex + metricLabels []string + + // wireIDCounter generates the ids sent on the wire, so concurrent + // requests with the same caller id don't collide in pending. The + // caller's id is restored on the response. + wireIDCounter atomic.Uint64 +} + +// WsSubscription ties an eth_subscribe or eth_unsubscribe request to the +// connection it is scoped to. The client only sends these methods when the +// request context carries one (see WithWsSubscription): a subscription +// created by a routed request would have no owner to receive or cancel it. +type WsSubscription struct { + // Handler receives the subscription's notifications (eth_subscribe + // only). It is registered before the next frame is read, so nothing + // sent right after the subscribe response is missed. + Handler func(params []byte) + // ID and Epoch are set when an eth_subscribe succeeds. eth_unsubscribe + // must carry the Epoch of the subscription it cancels; it is not sent + // once that connection is gone, since ids are reused across connections. + ID string + Epoch uint64 +} + +type wsSubscriptionCtxKey struct{} + +// WithWsSubscription returns ctx carrying sub for an eth_subscribe or +// eth_unsubscribe sent through SendRequest. +func WithWsSubscription(ctx context.Context, sub *WsSubscription) context.Context { + return context.WithValue(ctx, wsSubscriptionCtxKey{}, sub) +} + +type wsPending struct { + ch chan *wsPendingResult + // sub is set for eth_subscribe. If the caller gives up, the entry is + // kept (abandoned, guarded by pendingMu) so a late success is + // unsubscribed instead of leaked. + sub *WsSubscription + abandoned bool +} + +type wsPendingResult struct { + message []byte + err error +} + +// wsMessage holds the fields that tell a response (id) from a notification +// (method). +type wsMessage struct { + ID interface{} `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Params json.RawMessage `json:"params,omitempty"` +} + +type wsNotificationParams struct { + Subscription string `json:"subscription"` + Result json.RawMessage `json:"result"` +} + +func NewWsJsonRpcClient( + appCtx context.Context, + logger *zerolog.Logger, + projectId string, + upstream common.Upstream, + parsedUrl *url.URL, + jsonRpcCfg *common.JsonRpcUpstreamConfig, + extractor common.JsonRpcErrorExtractor, +) (ClientInterface, error) { + headers := http.Header{} + if jsonRpcCfg != nil { + for k, v := range jsonRpcCfg.Headers { + headers.Set(k, v) + } + } + + client := &WsJsonRpcClient{ + Url: parsedUrl, + headers: headers, + pingInterval: wsPingInterval, + pongWait: wsPongWait, + projectId: projectId, + upstream: upstream, + appCtx: appCtx, + logger: logger, + pending: make(map[string]*wsPending), + subHandlers: make(map[string]func(params []byte)), + onDisconnectCbs: make(map[string]func()), + onReconnectCbs: make(map[string]func()), + errorExtractor: extractor, + } + + if err := client.connect(); err != nil { + // readLoop keeps re-dialing in the background. + logger.Warn().Err(err).Str("url", parsedUrl.String()).Msg("initial websocket connection failed, will retry in background") + } + + go client.readLoop() + go client.pingLoop() + go func() { + <-appCtx.Done() + client.shutdown() + }() + + return client, nil +} + +func (c *WsJsonRpcClient) GetType() ClientType { + return ClientTypeWsJsonRpc +} + +func (c *WsJsonRpcClient) IsConnected() bool { + return c.epoch.Load() != 0 +} + +// Epoch identifies the current connection, or is 0 while disconnected. It +// changes on every reconnect. +func (c *WsJsonRpcClient) Epoch() uint64 { + return c.epoch.Load() +} + +func (c *WsJsonRpcClient) SendRequest(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + ctx, span := common.StartDetailSpan(ctx, "WsJsonRpcClient.SendRequest", + trace.WithAttributes( + attribute.String("upstream.id", c.upstream.Id()), + ), + ) + defer span.End() + + startedAt := time.Now() + + jrReq, err := req.JsonRpcRequest() + if err != nil { + return nil, common.NewErrUpstreamRequest( + err, + c.upstream, + req.NetworkId(), + "", + 0, 0, 0, 0, + ) + } + + wireID := c.wireIDCounter.Add(1) + idKey := strconv.FormatUint(wireID, 10) + + // Serialize the JSON-RPC request with the rewritten wire id + jrReq.RLock() + originalID := jrReq.ID + method := jrReq.Method + requestBody, err := common.SonicCfg.Marshal(map[string]interface{}{ + "jsonrpc": jrReq.JSONRPC, + "id": wireID, + "method": method, + "params": jrReq.Params, + }) + jrReq.RUnlock() + if err != nil { + common.SetTraceSpanError(span, err) + return nil, common.NewErrUpstreamRequest( + err, + c.upstream, + req.NetworkId(), + method, + 0, 0, 0, 0, + ) + } + + pending := &wsPending{ch: make(chan *wsPendingResult, 1)} + var unsubscribe *WsSubscription + if method == "eth_subscribe" || method == "eth_unsubscribe" { + sub, _ := ctx.Value(wsSubscriptionCtxKey{}).(*WsSubscription) + if sub == nil { + return nil, common.NewErrUpstreamRequestSkipped( + fmt.Errorf("%s is scoped to the upstream connection and only sent by its subscription owner", method), + c.upstream.Id(), + ) + } + if method == "eth_subscribe" { + pending.sub = sub + } else { + unsubscribe = sub + } + } + + // Register under connMu so the entry belongs to exactly this conn: + // teardown swaps conn and drains pending under the same lock. + c.connMu.Lock() + conn := c.conn + stale := unsubscribe != nil && unsubscribe.Epoch != c.epoch.Load() + if conn != nil && !stale { + c.pendingMu.Lock() + c.pending[idKey] = pending + c.pendingMu.Unlock() + } + c.connMu.Unlock() + + if stale { + // The subscription died with its connection; its id may since have + // been reused on the new one. + return nil, common.NewErrUpstreamRequestSkipped(errors.New("subscription's connection is already closed"), c.upstream.Id()) + } + if conn == nil { + // Re-dial in progress: fail fast so the request fails over. + err := common.NewErrEndpointTransportFailure(c.Url, errors.New("websocket connection not established")) + common.SetTraceSpanError(span, err) + return nil, err + } + + if err := c.writeToConn(conn, websocket.TextMessage, requestBody); err != nil { + c.pendingMu.Lock() + delete(c.pending, idKey) + c.pendingMu.Unlock() + common.SetTraceSpanError(span, err) + return nil, common.NewErrEndpointTransportFailure(c.Url, err) + } + + c.logger.Debug(). + Str("host", c.Url.Host). + RawJSON("request", requestBody). + Msg("sent json rpc websocket request") + + // Wait for response. A caller that gives up after readLoop or teardown + // already claimed the entry takes the result anyway: it is on its way, + // and dropping it would strand a subscribe it created. + var result *wsPendingResult + select { + case result = <-pending.ch: + case <-ctx.Done(): + if c.abandonPending(idKey) { + err := ctx.Err() + if errors.Is(err, context.DeadlineExceeded) { + err = common.NewErrEndpointRequestTimeout(time.Since(startedAt), err) + } else if errors.Is(err, context.Canceled) { + err = common.NewErrEndpointRequestCanceled(err) + } + common.SetTraceSpanError(span, err) + return nil, err + } + result = <-pending.ch + case <-c.appCtx.Done(): + if c.abandonPending(idKey) { + return nil, common.NewErrEndpointRequestCanceled(c.appCtx.Err()) + } + result = <-pending.ch + } + if result.err != nil { + common.SetTraceSpanError(span, result.err) + return nil, result.err + } + nr := common.NewNormalizedResponse().WithRequest(req).WithBody(io.NopCloser(bytes.NewReader(result.message))) + // Restore the caller's original JSON-RPC id on the response. + jrr, perr := nr.JsonRpcResponse(ctx) + if perr == nil && jrr != nil { + _ = jrr.SetID(originalID) + } + if err := classifyJsonRpcError(&http.Response{StatusCode: http.StatusOK, Header: http.Header{}}, nr, jrr, perr, c.errorExtractor, c.upstream); err != nil { + common.SetTraceSpanError(span, err) + return nr, err + } + return nr, nil +} + +// abandonPending drops a request whose caller gave up. A subscribe stays +// registered so readLoop can cancel it if the upstream still creates it. +func (c *WsJsonRpcClient) abandonPending(idKey string) bool { + c.pendingMu.Lock() + defer c.pendingMu.Unlock() + p, ok := c.pending[idKey] + if ok { + if p.sub != nil { + p.abandoned = true + } else { + delete(c.pending, idKey) + } + } + return ok +} + +// UnregisterSubscriptionHandler removes the handler of a subscription made +// on connection epoch. A no-op once that connection is gone: its handlers +// were dropped with it, and the id may be reused by the next connection. +func (c *WsJsonRpcClient) UnregisterSubscriptionHandler(upstreamSubID string, epoch uint64) { + c.subHandlersMu.Lock() + if c.epoch.Load() == epoch { + delete(c.subHandlers, upstreamSubID) + } + c.subHandlersMu.Unlock() +} + +// SetOnDisconnect sets the callback under id that runs when the connection +// drops. +func (c *WsJsonRpcClient) SetOnDisconnect(id string, callback func()) { + c.onDisconnectMu.Lock() + c.onDisconnectCbs[id] = callback + c.onDisconnectMu.Unlock() +} + +// SetOnReconnect sets the callback under id that runs after a reconnect. +func (c *WsJsonRpcClient) SetOnReconnect(id string, callback func()) { + c.onReconnectMu.Lock() + c.onReconnectCbs[id] = callback + c.onReconnectMu.Unlock() +} + +func (c *WsJsonRpcClient) connect() error { + dialer := websocket.Dialer{ + HandshakeTimeout: 10 * time.Second, + } + + if c.Url.Scheme == "wss" { + dialer.TLSClientConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + } + } + + conn, _, err := dialer.DialContext(c.appCtx, c.Url.String(), c.headers) + if err != nil { + return err + } + + // Every pong (here) and data frame (in readLoop) extends the liveness + // deadline. + _ = conn.SetReadDeadline(time.Now().Add(c.pongWait)) + conn.SetPongHandler(func(string) error { + return conn.SetReadDeadline(time.Now().Add(c.pongWait)) + }) + + c.connMu.Lock() + if c.appCtx.Err() != nil { + // shutdown already ran; don't install a connection behind it. + c.connMu.Unlock() + _ = conn.Close() + return c.appCtx.Err() + } + c.conn = conn + c.connectedAt = time.Now() + c.epochs++ + c.epoch.Store(c.epochs) + c.connMu.Unlock() + c.setConnectedMetric() + + c.logger.Info().Str("url", c.Url.String()).Msg("websocket connection established") + return nil +} + +// teardownConn closes conn, marks the client disconnected, fails every +// request pending on it and drops its subscription handlers. +func (c *WsJsonRpcClient) teardownConn(conn *websocket.Conn, cause error) { + c.connMu.Lock() + c.conn = nil + c.epoch.Store(0) + c.pendingMu.Lock() + pending := c.pending + c.pending = make(map[string]*wsPending) + c.pendingMu.Unlock() + c.connMu.Unlock() + + if conn != nil { + _ = conn.Close() + } + c.subHandlersMu.Lock() + clear(c.subHandlers) + c.subHandlersMu.Unlock() + c.setConnectedMetric() + for _, p := range pending { + p.ch <- &wsPendingResult{err: cause} + } +} + +// setConnectedMetric publishes the connectivity gauge. Labels are resolved +// on every publish (pingLoop republishes each tick) because the upstream's +// network label is assigned after the client is built; the series under +// the previous labels is deleted when they change. +func (c *WsJsonRpcClient) setConnectedMetric() { + labels := []string{c.projectId, c.upstream.VendorName(), c.upstream.NetworkLabel(), c.upstream.Id()} + + c.metricMu.Lock() + defer c.metricMu.Unlock() + if c.metricLabels != nil && !slices.Equal(c.metricLabels, labels) { + telemetry.MetricUpstreamWebsocketConnected.DeleteLabelValues(c.metricLabels...) + } + c.metricLabels = labels + v := 0.0 + if c.IsConnected() { + v = 1 + } + telemetry.MetricUpstreamWebsocketConnected.WithLabelValues(labels...).Set(v) +} + +// readLoop owns the connection lifecycle: it reads the current connection +// and, when there is none, re-dials with backoff. pingLoop only closes a +// broken connection, so a nil conn always has a re-dial behind it. +func (c *WsJsonRpcClient) readLoop() { + backoff := wsReconnectMin + for { + if c.appCtx.Err() != nil { + return + } + + c.connMu.Lock() + conn, connectedAt, epoch := c.conn, c.connectedAt, c.epoch.Load() + c.connMu.Unlock() + + if conn == nil { + // Always wait before dialing so a peer that accepts and then + // immediately drops the connection can't drive a hot loop. + wait := backoff/2 + rand.N(backoff/2+1) + backoff = min(time.Duration(float64(backoff)*wsReconnectFactor), wsReconnectMax) + c.logger.Info().Dur("backoff", wait).Msg("attempting websocket reconnection") + select { + case <-time.After(wait): + case <-c.appCtx.Done(): + return + } + if err := c.connect(); err != nil { + c.logger.Warn().Err(err).Msg("websocket reconnection failed") + continue + } + c.logger.Info().Msg("websocket reconnected successfully") + c.fireCallbacks(&c.onReconnectMu, c.onReconnectCbs) + continue + } + + _, message, err := conn.ReadMessage() + if err != nil { + if c.appCtx.Err() != nil { + return + } + var netErr net.Error + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + c.logger.Info().Msg("websocket connection closed normally") + } else if errors.As(err, &netErr) && netErr.Timeout() { + c.logger.Warn().Err(err).Dur("pongWait", c.pongWait). + Msg("websocket peer silent beyond liveness deadline, reconnecting") + } else { + c.logger.Warn().Err(err).Msg("websocket read error, will reconnect") + } + // Only a connection that outlived a liveness window resets the + // backoff; one dropped right after the handshake keeps growing it. + if time.Since(connectedAt) >= c.pongWait { + backoff = wsReconnectMin + } + c.teardownConn(conn, common.NewErrEndpointTransportFailure(c.Url, fmt.Errorf("websocket connection lost: %w", err))) + c.fireCallbacks(&c.onDisconnectMu, c.onDisconnectCbs) + continue + } + + _ = conn.SetReadDeadline(time.Now().Add(c.pongWait)) + + c.handleMessage(message, epoch) + } +} + +// fireCallbacks snapshots the callback map under rlock and invokes each +// callback synchronously, so disconnect callbacks always complete before +// the reconnect callbacks of the next connection run. Callbacks must +// therefore be fast and must not block on the WS client's request path. +func (c *WsJsonRpcClient) fireCallbacks(mu *sync.RWMutex, cbs map[string]func()) { + mu.RLock() + snapshot := make([]func(), 0, len(cbs)) + for _, cb := range cbs { + snapshot = append(snapshot, cb) + } + mu.RUnlock() + for _, cb := range snapshot { + func() { + defer c.recoverPanic("ws-upstream-callback") + cb() + }() + } +} + +// recoverPanic keeps a panic in a callback or a message handler from +// crashing the process. The read loop carries on with the next frame: the +// panic comes from the handler, not the connection, so reconnecting would +// only repeat it. +func (c *WsJsonRpcClient) recoverPanic(scope string) { + if rec := recover(); rec != nil { + telemetry.MetricUnexpectedPanicTotal.WithLabelValues( + scope, + fmt.Sprintf("project:%s upstream:%s", c.projectId, c.upstream.Id()), + common.ErrorFingerprint(rec), + ).Inc() + c.logger.Error(). + Interface("panic", rec). + Str("stack", string(debug.Stack())). + Msg("unexpected panic in websocket upstream client") + } +} + +// handleMessage dispatches one frame read from connection epoch. +func (c *WsJsonRpcClient) handleMessage(message []byte, epoch uint64) { + defer c.recoverPanic("ws-upstream-message") + var msg wsMessage + if err := common.SonicCfg.Unmarshal(message, &msg); err != nil { + c.logger.Warn().Err(err).Str("raw", string(message)).Msg("failed to parse websocket message") + return + } + + if msg.Method != "" && msg.ID == nil { + c.handleNotification(msg.Method, msg.Params, epoch) + return + } + + if msg.ID != nil { + idKey := normalizeIDKey(msg.ID) + + c.pendingMu.Lock() + p, ok := c.pending[idKey] + delete(c.pending, idKey) + abandoned := ok && p.abandoned + c.pendingMu.Unlock() + + if !ok { + c.logger.Debug().Str("id", idKey).Msg("received response for unknown request ID") + return + } + var subID string + if p.sub != nil && common.SonicCfg.Unmarshal(msg.Result, &subID) == nil && subID != "" { + if abandoned { + go c.unsubscribeOrphan(subID, epoch) + return + } + // The id outlives the frame it was decoded from. + subID = strings.Clone(subID) + // Register before the next frame is read so no notification + // is dropped as belonging to an unknown subscription. + c.subHandlersMu.Lock() + c.subHandlers[subID] = p.sub.Handler + c.subHandlersMu.Unlock() + p.sub.ID, p.sub.Epoch = subID, epoch + } + if !abandoned { + p.ch <- &wsPendingResult{message: message} + } + return + } + + c.logger.Debug().Str("raw", string(message)).Msg("received unhandled websocket message") +} + +func (c *WsJsonRpcClient) handleNotification(method string, params []byte, epoch uint64) { + if method != "eth_subscription" { + c.logger.Debug().Str("method", method).Msg("received non-subscription notification") + return + } + + var notifParams wsNotificationParams + if err := common.SonicCfg.Unmarshal(params, ¬ifParams); err != nil { + c.logger.Warn().Err(err).Msg("failed to parse subscription notification params") + return + } + + c.subHandlersMu.RLock() + handler, ok := c.subHandlers[notifParams.Subscription] + c.subHandlersMu.RUnlock() + + if !ok { + c.logger.Debug().Str("subscriptionId", notifParams.Subscription).Msg("received notification for unknown subscription") + // Nothing owns it: cancel it, unless a subscribe in flight may be + // about to claim it. Each later notification retries the cancel. + if notifParams.Subscription != "" && !c.subscribeInFlight() { + go c.unsubscribeOrphan(strings.Clone(notifParams.Subscription), epoch) + } + return + } + + handler(params) +} + +// subscribeInFlight reports whether an eth_subscribe still has a caller +// waiting for its response. +func (c *WsJsonRpcClient) subscribeInFlight() bool { + c.pendingMu.Lock() + defer c.pendingMu.Unlock() + for _, p := range c.pending { + if p.sub != nil && !p.abandoned { + return true + } + } + return false +} + +// writeToConn writes to an explicit connection so callers that need to act +// on a write failure (e.g. pingLoop closing the broken conn) operate on the +// exact connection they wrote to, not whatever c.conn points at by then. +func (c *WsJsonRpcClient) writeToConn(conn *websocket.Conn, messageType int, data []byte) error { + c.writeMu.Lock() + defer c.writeMu.Unlock() + + if err := conn.SetWriteDeadline(time.Now().Add(wsWriteWait)); err != nil { + return err + } + return conn.WriteMessage(messageType, data) +} + +// unsubscribeOrphan cancels a subscription the upstream created after its +// caller gave up. Fire-and-forget: the response is dropped as unknown. +func (c *WsJsonRpcClient) unsubscribeOrphan(subID string, epoch uint64) { + body, err := common.SonicCfg.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "id": c.wireIDCounter.Add(1), + "method": "eth_unsubscribe", + "params": []string{subID}, + }) + if err != nil { + return + } + c.connMu.Lock() + conn := c.conn + live := c.epoch.Load() == epoch + c.connMu.Unlock() + if conn != nil && live { + _ = c.writeToConn(conn, websocket.TextMessage, body) + } +} + +func (c *WsJsonRpcClient) pingLoop() { + ticker := time.NewTicker(c.pingInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + c.setConnectedMetric() + c.connMu.Lock() + conn := c.conn + c.connMu.Unlock() + if conn == nil { + continue + } + if err := c.writeToConn(conn, websocket.PingMessage, nil); err != nil { + // Only close it: readLoop's ReadMessage then fails and it + // tears down and re-dials. Clearing c.conn here instead would + // leave nobody to re-dial if readLoop was busy in a handler. + c.logger.Warn().Err(err).Msg("websocket ping write failed, closing connection to force reconnect") + _ = conn.Close() + } + case <-c.appCtx.Done(): + return + } + } +} + +// normalizeIDKey converts a decoded JSON-RPC id to the key used in pending. +// Numeric ids decode as float64, which %v would print in scientific +// notation. +func normalizeIDKey(id interface{}) string { + switch v := id.(type) { + case float64: + return fmt.Sprintf("%.0f", v) + case int: + return fmt.Sprintf("%d", v) + case int64: + return fmt.Sprintf("%d", v) + case string: + return v + default: + return fmt.Sprintf("%v", v) + } +} + +func (c *WsJsonRpcClient) shutdown() { + c.connMu.Lock() + conn := c.conn + c.connMu.Unlock() + + if conn != nil { + _ = conn.WriteControl( + websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), + time.Now().Add(wsWriteWait), + ) + } + c.teardownConn(conn, common.NewErrEndpointRequestCanceled(fmt.Errorf("websocket client shutting down"))) +} diff --git a/clients/ws_json_rpc_client_errors_test.go b/clients/ws_json_rpc_client_errors_test.go new file mode 100644 index 000000000..1c25b1b85 --- /dev/null +++ b/clients/ws_json_rpc_client_errors_test.go @@ -0,0 +1,76 @@ +package clients_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/erpc/erpc/clients" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/upstream" + "github.com/gorilla/websocket" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// TestWsClientNormalizesJsonRpcErrorsLikeHttp: the same upstream error body +// must normalize to the same error code over WS as over HTTP. +func TestWsClientNormalizesJsonRpcErrorsLikeHttp(t *testing.T) { + const errBody = `,"error":{"code":3,"message":"execution reverted","data":"0x08c379a0"}}` + + httpSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1` + errBody)) + })) + t.Cleanup(httpSrv.Close) + + upgrader := websocket.Upgrader{} + wsSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + for { + _, msg, err := conn.ReadMessage() + if err != nil { + return + } + var req struct { + ID json.RawMessage `json:"id"` + } + _ = common.SonicCfg.Unmarshal(msg, &req) + _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","id":`+string(req.ID)+errBody)) + } + })) + t.Cleanup(wsSrv.Close) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + logger := zerolog.New(zerolog.NewTestWriter(t)).Level(zerolog.WarnLevel) + up := common.NewFakeUpstream("test-upstream") + extractor := upstream.NewCompositeJsonRpcErrorExtractor() + + httpURL, err := url.Parse(httpSrv.URL) + require.NoError(t, err) + httpClient, err := clients.NewGenericHttpJsonRpcClient(ctx, &logger, "test-project", up, httpURL, nil, nil, extractor) + require.NoError(t, err) + + wsURL, err := url.Parse(wsSrv.URL) + require.NoError(t, err) + wsURL.Scheme = "ws" + wsClient, err := clients.NewWsJsonRpcClient(ctx, &logger, "test-project", up, wsURL, nil, extractor) + require.NoError(t, err) + + for name, c := range map[string]clients.ClientInterface{"http": httpClient, "ws": wsClient} { + reqCtx, reqCancel := context.WithTimeout(ctx, 2*time.Second) + _, err := c.SendRequest(reqCtx, common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[]}`))) + reqCancel() + require.Error(t, err, name) + require.True(t, common.HasErrorCode(err, common.ErrCodeEndpointExecutionException), "%s: got %v", name, err) + } +} diff --git a/clients/ws_json_rpc_client_test.go b/clients/ws_json_rpc_client_test.go new file mode 100644 index 000000000..ff629cb10 --- /dev/null +++ b/clients/ws_json_rpc_client_test.go @@ -0,0 +1,698 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/gorilla/websocket" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeWsServer is a minimal JSON-RPC WebSocket upstream whose connections +// can be black-holed: TCP stays open but nothing, not even a pong, is +// written back. +type fakeWsServer struct { + t *testing.T + srv *httptest.Server + + mu sync.Mutex + conns []*fakeWsConn + + newConn chan *fakeWsConn +} + +type fakeWsConn struct { + conn *websocket.Conn + writeMu sync.Mutex + // silent simulates a black-holed path: pings are swallowed (no pong) + // and the server never writes, but the TCP connection stays open. + silent atomic.Bool + // subscribeCh receives the request id (raw JSON) of each + // eth_subscribe request the server answers. + subscribeCh chan string +} + +func newFakeWsServer(t *testing.T) *fakeWsServer { + f := &fakeWsServer{t: t, newConn: make(chan *fakeWsConn, 16)} + upgrader := websocket.Upgrader{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + sc := &fakeWsConn{conn: conn, subscribeCh: make(chan string, 16)} + conn.SetPingHandler(func(appData string) error { + if sc.silent.Load() { + return nil // swallow: black-holed path sends no pong + } + return conn.WriteControl(websocket.PongMessage, []byte(appData), time.Now().Add(time.Second)) + }) + f.mu.Lock() + f.conns = append(f.conns, sc) + f.mu.Unlock() + f.newConn <- sc + go sc.readLoop() + })) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeWsServer) wsURL(t *testing.T) *url.URL { + u, err := url.Parse(f.srv.URL) + require.NoError(t, err) + u.Scheme = "ws" + return u +} + +// readLoop answers eth_subscribe with an incrementing subscription id. +// Control frames (pings) are handled inside ReadMessage via the handler +// installed above, so silencing the ping handler is enough to emulate a +// peer that no longer processes anything. +func (sc *fakeWsConn) readLoop() { + subCounter := 0 + for { + _, msg, err := sc.conn.ReadMessage() + if err != nil { + return + } + if sc.silent.Load() { + continue // black-holed: never respond + } + var req struct { + ID interface{} `json:"id"` + Method string `json:"method"` + Params []interface{} `json:"params"` + } + if err := common.SonicCfg.Unmarshal(msg, &req); err != nil { + continue + } + if req.Method == "eth_subscribe" { + subCounter++ + subID := "0xtestsub" + string(rune('0'+subCounter)) + resp, _ := common.SonicCfg.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "id": req.ID, + "result": subID, + }) + sc.write(websocket.TextMessage, resp) + sc.subscribeCh <- subID + } + } +} + +func (sc *fakeWsConn) write(messageType int, data []byte) { + sc.writeMu.Lock() + defer sc.writeMu.Unlock() + _ = sc.conn.SetWriteDeadline(time.Now().Add(time.Second)) + _ = sc.conn.WriteMessage(messageType, data) +} + +func (sc *fakeWsConn) sendNewHead(subID string, blockNumberHex string) { + notif, _ := common.SonicCfg.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": map[string]interface{}{ + "subscription": subID, + "result": map[string]interface{}{ + "number": blockNumberHex, + "hash": "0xhash" + blockNumberHex, + "parentHash": "0xparent" + blockNumberHex, + }, + }, + }) + sc.write(websocket.TextMessage, notif) +} + +// compressWsLiveness shrinks the keepalive windows so dead-peer detection +// happens in milliseconds instead of minutes, restoring them on cleanup. +func compressWsLiveness(t *testing.T) { + origPing, origPong := wsPingInterval, wsPongWait + wsPingInterval = 50 * time.Millisecond + wsPongWait = 150 * time.Millisecond + t.Cleanup(func() { + wsPingInterval, wsPongWait = origPing, origPong + }) +} + +func newTestWsClient(t *testing.T, u *url.URL) *WsJsonRpcClient { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + logger := zerolog.New(zerolog.NewTestWriter(t)).Level(zerolog.WarnLevel) + up := common.NewFakeUpstream("test-ws-upstream") + ci, err := NewWsJsonRpcClient(ctx, &logger, "test-project", up, u, nil, nil) + require.NoError(t, err) + c, ok := ci.(*WsJsonRpcClient) + require.True(t, ok) + return c +} + +func subscribeNewHeads(t *testing.T, c *WsJsonRpcClient, handler func(params []byte)) string { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + sub := &WsSubscription{Handler: handler} + nq := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`)) + _, err := c.SendRequest(WithWsSubscription(ctx, sub), nq) + require.NoError(t, err) + require.NotEmpty(t, sub.ID) + require.Equal(t, c.Epoch(), sub.Epoch) + return sub.ID +} + +// TestWsClientDetectsSilentPeerAndReconnects: a peer that keeps TCP open +// but stops responding must be declared dead by the liveness deadline, +// re-dialed, and resume delivering notifications. +func TestWsClientDetectsSilentPeerAndReconnects(t *testing.T) { + compressWsLiveness(t) + server := newFakeWsServer(t) + client := newTestWsClient(t, server.wsURL(t)) + + disconnected := make(chan struct{}, 1) + reconnected := make(chan struct{}, 1) + client.SetOnDisconnect("test", func() { + select { + case disconnected <- struct{}{}: + default: + } + }) + client.SetOnReconnect("test", func() { + select { + case reconnected <- struct{}{}: + default: + } + }) + + // First connection established and subscribed. + var conn1 *fakeWsConn + select { + case conn1 = <-server.newConn: + case <-time.After(2 * time.Second): + t.Fatal("server never saw the initial connection") + } + heads := make(chan []byte, 16) + onHead := func(params []byte) { heads <- params } + subID1 := subscribeNewHeads(t, client, onHead) + conn1.sendNewHead(subID1, "0x1") + select { + case <-heads: + case <-time.After(2 * time.Second): + t.Fatal("never received the first head") + } + + // Black-hole the connection: TCP stays open, nothing flows back. + conn1.silent.Store(true) + + select { + case <-disconnected: + case <-time.After(3 * time.Second): + t.Fatal("client never detected the silent (half-open) connection — liveness deadline did not fire") + } + + select { + case <-reconnected: + case <-time.After(3 * time.Second): + t.Fatal("client never reconnected after detecting the dead connection") + } + + var conn2 *fakeWsConn + select { + case conn2 = <-server.newConn: + case <-time.After(2 * time.Second): + t.Fatal("server never saw the re-dialed connection") + } + assert.True(t, client.IsConnected()) + + // Re-subscribe on the new connection (the wsupstream adapter does this + // from its reconnect hook) and verify notifications flow again. + subID2 := subscribeNewHeads(t, client, onHead) + conn2.sendNewHead(subID2, "0x2") + select { + case <-heads: + case <-time.After(2 * time.Second): + t.Fatal("no heads delivered after reconnection — client did not self-heal") + } +} + +// TestWsClientPingWriteFailureForcesReconnect covers the secondary path: +// when the ping write itself errors (connection reset under our feet), the +// client must tear the connection down and re-dial rather than only logging. +func TestWsClientPingWriteFailureForcesReconnect(t *testing.T) { + compressWsLiveness(t) + server := newFakeWsServer(t) + client := newTestWsClient(t, server.wsURL(t)) + + reconnected := make(chan struct{}, 1) + client.SetOnReconnect("test", func() { + select { + case reconnected <- struct{}{}: + default: + } + }) + + var conn1 *fakeWsConn + select { + case conn1 = <-server.newConn: + case <-time.After(2 * time.Second): + t.Fatal("server never saw the initial connection") + } + + // Hard-kill the server side of the TCP connection (RST-ish): the next + // client ping write (or read) fails. + _ = conn1.conn.UnderlyingConn().Close() + + select { + case <-reconnected: + case <-time.After(3 * time.Second): + t.Fatal("client never reconnected after the connection was killed") + } + select { + case <-server.newConn: + case <-time.After(2 * time.Second): + t.Fatal("server never saw the re-dialed connection") + } +} + +// relabelledUpstream reports a network label that can change after the +// client is built, as upstream.Upstream's does once it joins a network. +type relabelledUpstream struct { + common.Upstream + label atomic.Value +} + +func (u *relabelledUpstream) NetworkLabel() string { return u.label.Load().(string) } + +// TestWsClientConnectedGaugeFollowsNetworkLabel: the connectivity gauge +// must move to the upstream's current network label and drop the series +// published under the old one. +func TestWsClientConnectedGaugeFollowsNetworkLabel(t *testing.T) { + compressWsLiveness(t) + server := newFakeWsServer(t) + up := &relabelledUpstream{Upstream: common.NewFakeUpstream("test-ws-gauge-upstream")} + up.label.Store("n/a") + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + logger := zerolog.New(zerolog.NewTestWriter(t)).Level(zerolog.WarnLevel) + _, err := NewWsJsonRpcClient(ctx, &logger, "test-project", up, server.wsURL(t), nil, nil) + require.NoError(t, err) + <-server.newConn + + gauge := telemetry.MetricUpstreamWebsocketConnected + labels := func(network string) []string { + return []string{"test-project", up.VendorName(), network, up.Id()} + } + require.Equal(t, 1.0, testutil.ToFloat64(gauge.WithLabelValues(labels("n/a")...))) + + up.label.Store("evm:123") + require.Eventually(t, func() bool { + return testutil.ToFloat64(gauge.WithLabelValues(labels("evm:123")...)) == 1 + }, 2*time.Second, 10*time.Millisecond) + assert.False(t, gauge.DeleteLabelValues(labels("n/a")...), "series under the stale label should have been deleted") +} + +// newWsTestServer upgrades every request and hands the connection to +// handle, closing it when handle returns. +func newWsTestServer(t *testing.T, handle func(conn *websocket.Conn)) *url.URL { + upgrader := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + handle(conn) + })) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + require.NoError(t, err) + u.Scheme = "ws" + return u +} + +// TestWsClientRedialsWhenConnDiesDuringHandler: the connection dies while +// readLoop is inside a subscription handler and the ping loop notices +// first. readLoop must still tear down and re-dial once the handler +// returns. +func TestWsClientRedialsWhenConnDiesDuringHandler(t *testing.T) { + compressWsLiveness(t) + server := newFakeWsServer(t) + client := newTestWsClient(t, server.wsURL(t)) + conn1 := <-server.newConn + + entered := make(chan struct{}) + release := make(chan struct{}) + subID := subscribeNewHeads(t, client, func([]byte) { + close(entered) + <-release + }) + conn1.sendNewHead(subID, "0x1") + <-entered + + _ = conn1.conn.UnderlyingConn().Close() + time.Sleep(3 * client.pingInterval) // a ping write fails while readLoop is busy + close(release) + + select { + case <-server.newConn: + case <-time.After(3 * time.Second): + t.Fatal("client never re-dialed after the connection died during a handler") + } +} + +// TestWsClientBacksOffWhenPeerDropsAfterHandshake: a peer that accepts the +// handshake and immediately closes must not drive a hot re-dial loop. +func TestWsClientBacksOffWhenPeerDropsAfterHandshake(t *testing.T) { + var handshakes atomic.Int64 + u := newWsTestServer(t, func(conn *websocket.Conn) { + handshakes.Add(1) + _ = conn.WriteControl(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.ClosePolicyViolation, ""), time.Now().Add(time.Second)) + }) + newTestWsClient(t, u) + + time.Sleep(time.Second) + assert.LessOrEqual(t, handshakes.Load(), int64(3)) +} + +// TestWsClientFailsFastWhileRedialing: while a re-dial is stuck in a slow +// handshake, requests must fail fast with a retryable error. +func TestWsClientFailsFastWhileRedialing(t *testing.T) { + var requests atomic.Int64 + first := make(chan *websocket.Conn, 1) + upgrader := websocket.Upgrader{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) > 1 { + time.Sleep(3 * time.Second) // never upgrade within the test + return + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + first <- conn + })) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + require.NoError(t, err) + u.Scheme = "ws" + client := newTestWsClient(t, u) + + _ = (<-first).UnderlyingConn().Close() + require.Eventually(t, func() bool { return requests.Load() >= 2 }, 3*time.Second, 5*time.Millisecond) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + start := time.Now() + _, err = client.SendRequest(ctx, common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`))) + require.Error(t, err) + assert.Less(t, time.Since(start), time.Second) + assert.True(t, common.HasErrorCode(err, common.ErrCodeEndpointTransportFailure), "got %v", err) +} + +// TestWsClientIgnoresDuplicateResponses: repeated responses for one id must +// not wedge readLoop. +func TestWsClientIgnoresDuplicateResponses(t *testing.T) { + u := newWsTestServer(t, func(conn *websocket.Conn) { + for { + _, msg, err := conn.ReadMessage() + if err != nil { + return + } + var req struct { + ID interface{} `json:"id"` + Method string `json:"method"` + } + _ = common.SonicCfg.Unmarshal(msg, &req) + resp, _ := common.SonicCfg.Marshal(map[string]interface{}{"jsonrpc": "2.0", "id": req.ID, "result": "0x1"}) + n := 1 + if req.Method == "dup" { + n = 50 + } + for i := 0; i < n; i++ { + _ = conn.WriteMessage(websocket.TextMessage, resp) + } + } + }) + client := newTestWsClient(t, u) + + send := func(method string) error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err := client.SendRequest(ctx, common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"`+method+`","params":[]}`))) + return err + } + for i := 0; i < 50; i++ { + require.NoError(t, send("dup")) + } + require.NoError(t, send("eth_chainId")) +} + +type wsTestRequest struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params []interface{} `json:"params"` +} + +// newSubscribeServer answers each eth_subscribe after delay with id +// "0x" from a per-connection counter, immediately followed by one +// notification for it. Other methods get result true. Every request read +// is reported on requests. +func newSubscribeServer(t *testing.T, delay time.Duration, requests chan<- wsTestRequest) *url.URL { + return newWsTestServer(t, func(conn *websocket.Conn) { + subs := 0 + for { + _, msg, err := conn.ReadMessage() + if err != nil { + return + } + var req wsTestRequest + _ = common.SonicCfg.Unmarshal(msg, &req) + requests <- req + if req.Method != "eth_subscribe" { + _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","id":`+string(req.ID)+`,"result":true}`)) + continue + } + time.Sleep(delay) + subs++ + subID := fmt.Sprintf("0x%x", subs) + _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","id":`+string(req.ID)+`,"result":"`+subID+`"}`)) + _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","method":"eth_subscription","params":{"subscription":"`+subID+`","result":"first"}}`)) + } + }) +} + +func sendSubscriptionRPC(ctx context.Context, c *WsJsonRpcClient, sub *WsSubscription, method string, params string) error { + if sub != nil { + ctx = WithWsSubscription(ctx, sub) + } + _, err := c.SendRequest(ctx, common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"`+method+`","params":`+params+`}`))) + return err +} + +// TestWsClientDeliversNotificationRightAfterSubscribe: a notification sent +// right behind the subscribe response must reach the handler. +func TestWsClientDeliversNotificationRightAfterSubscribe(t *testing.T) { + requests := make(chan wsTestRequest, 16) + client := newTestWsClient(t, newSubscribeServer(t, 0, requests)) + + got := make(chan string, 1) + sub := &WsSubscription{Handler: func(params []byte) { got <- string(params) }} + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + require.NoError(t, sendSubscriptionRPC(ctx, client, sub, "eth_subscribe", `["logs",{}]`)) + assert.Equal(t, "0x1", sub.ID) + + select { + case params := <-got: + assert.Contains(t, params, `"first"`) + case <-time.After(2 * time.Second): + t.Fatal("notification sent right after the subscribe response was dropped") + } +} + +// TestWsClientRejectsUnownedSubscriptionMethods: a routed eth_subscribe or +// eth_unsubscribe (no WsSubscription in ctx) would create or cancel +// connection state nobody owns; it must be skipped without reaching the +// upstream. +func TestWsClientRejectsUnownedSubscriptionMethods(t *testing.T) { + requests := make(chan wsTestRequest, 16) + client := newTestWsClient(t, newSubscribeServer(t, 0, requests)) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + for _, method := range []string{"eth_subscribe", "eth_unsubscribe"} { + err := sendSubscriptionRPC(ctx, client, nil, method, `["0x1"]`) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRequestSkipped), "%s: got %v", method, err) + assert.False(t, common.IsRetryableTowardsUpstream(err), method) + } + require.NoError(t, sendSubscriptionRPC(ctx, client, nil, "eth_chainId", `[]`)) + assert.Equal(t, "eth_chainId", (<-requests).Method, "the subscription methods must not have been sent") +} + +// TestWsClientUnsubscribesAbandonedSubscribe: when the caller times out +// but the upstream still creates the subscription, the client cancels it. +func TestWsClientUnsubscribesAbandonedSubscribe(t *testing.T) { + requests := make(chan wsTestRequest, 16) + client := newTestWsClient(t, newSubscribeServer(t, 300*time.Millisecond, requests)) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + sub := &WsSubscription{Handler: func([]byte) { t.Error("abandoned subscription delivered a notification") }} + require.Error(t, sendSubscriptionRPC(ctx, client, sub, "eth_subscribe", `["newHeads"]`)) + + assert.Equal(t, "eth_subscribe", (<-requests).Method) + select { + case req := <-requests: + assert.Equal(t, "eth_unsubscribe", req.Method) + assert.Equal(t, []interface{}{"0x1"}, req.Params) + case <-time.After(2 * time.Second): + t.Fatal("late subscription was never unsubscribed") + } +} + +// TestWsClientSubscriptionsAreConnectionScoped: ids restart per connection, +// so after a reconnect an old handler must not receive a reused id's +// notifications, and an old subscription must not be unsubscribed on the +// new connection. +func TestWsClientSubscriptionsAreConnectionScoped(t *testing.T) { + compressWsLiveness(t) + server := newFakeWsServer(t) + client := newTestWsClient(t, server.wsURL(t)) + conn1 := <-server.newConn + + old := make(chan struct{}, 1) + subID := subscribeNewHeads(t, client, func([]byte) { old <- struct{}{} }) + <-conn1.subscribeCh + oldEpoch := client.Epoch() + + _ = conn1.conn.UnderlyingConn().Close() + var conn2 *fakeWsConn + select { + case conn2 = <-server.newConn: + case <-time.After(3 * time.Second): + t.Fatal("client never re-dialed") + } + require.Eventually(t, client.IsConnected, time.Second, 5*time.Millisecond) + require.NotEqual(t, oldEpoch, client.Epoch()) + + conn2.sendNewHead(subID, "0x2") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + err := sendSubscriptionRPC(ctx, client, &WsSubscription{Epoch: oldEpoch}, "eth_unsubscribe", `["`+subID+`"]`) + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamRequestSkipped), "got %v", err) + select { + case <-old: + t.Fatal("handler from the previous connection received a notification") + case <-time.After(100 * time.Millisecond): + } +} + +// A panic in a subscription handler must not crash the process or end the +// read loop: the next notification on the same connection still arrives. +func TestWsClientSurvivesPanickingHandler(t *testing.T) { + server := newFakeWsServer(t) + client := newTestWsClient(t, server.wsURL(t)) + conn := <-server.newConn + + var calls atomic.Int32 + second := make(chan struct{}) + subID := subscribeNewHeads(t, client, func([]byte) { + if calls.Add(1) == 1 { + panic("handler bug") + } + close(second) + }) + epoch := client.Epoch() + + conn.sendNewHead(subID, "0x1") + conn.sendNewHead(subID, "0x2") + select { + case <-second: + case <-time.After(3 * time.Second): + t.Fatal("notification after a handler panic never arrived") + } + assert.Equal(t, epoch, client.Epoch(), "a handler panic must not reconnect") +} + +// A panic in one reconnect callback must not stop the others or the +// reconnect. +func TestWsClientSurvivesPanickingCallback(t *testing.T) { + compressWsLiveness(t) + server := newFakeWsServer(t) + client := newTestWsClient(t, server.wsURL(t)) + conn1 := <-server.newConn + + reconnected := make(chan struct{}, 4) + client.SetOnReconnect("bad", func() { panic("callback bug") }) + client.SetOnReconnect("good", func() { reconnected <- struct{}{} }) + + _ = conn1.conn.UnderlyingConn().Close() + select { + case <-server.newConn: + case <-time.After(3 * time.Second): + t.Fatal("client never re-dialed") + } + select { + case <-reconnected: + case <-time.After(3 * time.Second): + t.Fatal("the other reconnect callback never ran") + } +} + +// A caller that gives up after readLoop claimed its subscribe response must +// still take the result, so the subscription isn't left live and owned by +// nobody. +func TestWsClientSubscribeClaimedThenCallerCancels(t *testing.T) { + requests := make(chan wsTestRequest, 16) + client := newTestWsClient(t, newSubscribeServer(t, 0, requests)) + require.Eventually(t, client.IsConnected, 2*time.Second, 5*time.Millisecond) + + sub := &WsSubscription{Handler: func([]byte) {}} + ctx, cancel := context.WithCancel(context.Background()) + // Stall handleMessage right after it deleted the pending entry and + // before it registers the handler / pushes the result. + client.subHandlersMu.Lock() + errCh := make(chan error, 1) + go func() { errCh <- sendSubscriptionRPC(ctx, client, sub, "eth_subscribe", `["newHeads"]`) }() + assert.Equal(t, "eth_subscribe", (<-requests).Method) + require.Eventually(t, func() bool { + client.pendingMu.Lock() + defer client.pendingMu.Unlock() + return len(client.pending) == 0 + }, 2*time.Second, time.Millisecond) + cancel() + time.Sleep(50 * time.Millisecond) + client.subHandlersMu.Unlock() + err := <-errCh + + registered := func() bool { + client.subHandlersMu.RLock() + defer client.subHandlersMu.RUnlock() + _, ok := client.subHandlers["0x1"] + return ok + } + require.Eventually(t, registered, time.Second, time.Millisecond, "handleMessage registers the handler once unblocked") + if err != nil { + var unsubscribed bool + select { + case req := <-requests: + unsubscribed = req.Method == "eth_unsubscribe" + case <-time.After(300 * time.Millisecond): + } + t.Fatalf("caller got %v, yet sub 0x1 handler registered=%v and unsubscribed=%v (stranded)", err, registered(), unsubscribed) + } + assert.Equal(t, "0x1", sub.ID, "a completed subscribe must reach its caller") + assert.True(t, registered()) +} diff --git a/go.mod b/go.mod index 9ed6dfc33..0c9035a29 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/go-logr/zerologr v1.2.3 github.com/go-redsync/redsync/v4 v4.17.0 github.com/golang-jwt/jwt/v4 v4.5.2 + github.com/gorilla/websocket v1.5.3 github.com/grafana/sobek v0.0.0-20241024150027-d91f02b05e9b github.com/h2non/gock v1.2.0 github.com/jackc/pgconn v1.14.3 diff --git a/go.sum b/go.sum index b225b7791..0beed6c53 100644 --- a/go.sum +++ b/go.sum @@ -192,6 +192,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafana/sobek v0.0.0-20241024150027-d91f02b05e9b h1:hzfIt1lf19Zx1jIYdeHvuWS266W+jL+7dxbpvH2PZMQ= github.com/grafana/sobek v0.0.0-20241024150027-d91f02b05e9b/go.mod h1:FmcutBFPLiGgroH42I4/HBahv7GxVjODcVWFTw1ISes= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= diff --git a/upstream/registry.go b/upstream/registry.go index c80df0368..a77a9273b 100644 --- a/upstream/registry.go +++ b/upstream/registry.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/url" "strings" "sync" "time" @@ -425,6 +426,47 @@ func (u *UpstreamsRegistry) GetNetworkUpstreams(ctx context.Context, networkId s return cp } +// GetFallbackEscapeUpstreams returns the network's bootstrapped fallback-tier +// upstreams that are not down for method and allow it. Selection-policy +// cordons are deliberately ignored: the per-request escape exists to get past +// them. +func (u *UpstreamsRegistry) GetFallbackEscapeUpstreams(ctx context.Context, networkId, method string) []*Upstream { + all := u.GetNetworkUpstreams(ctx, networkId) + out := make([]*Upstream, 0, len(all)) + for _, up := range all { + cfg := up.Config() + if cfg == nil || !cfg.HasTag(common.TagTierFallback) { + continue + } + if up.IsDown(method) { + continue + } + if allowed, err := up.ShouldHandleMethod(method); err != nil || !allowed { + continue + } + out = append(out, up) + } + return out +} + +// GetWsUpstreams returns all WS-capable upstreams for a network (ws:// or wss:// endpoints). +func (u *UpstreamsRegistry) GetWsUpstreams(ctx context.Context, networkId string) []*Upstream { + all := u.GetNetworkUpstreams(ctx, networkId) + var ws []*Upstream + for _, up := range all { + if cfg := up.Config(); cfg != nil && IsWsEndpoint(cfg.Endpoint) { + ws = append(ws, up) + } + } + return ws +} + +// IsWsEndpoint reports whether endpoint uses the ws:// or wss:// scheme. +func IsWsEndpoint(endpoint string) bool { + parsed, err := url.Parse(endpoint) + return err == nil && (parsed.Scheme == "ws" || parsed.Scheme == "wss") +} + func (u *UpstreamsRegistry) GetAllUpstreams() []*Upstream { u.upstreamsMu.RLock() defer u.upstreamsMu.RUnlock() diff --git a/upstream/registry_fallback_escape_test.go b/upstream/registry_fallback_escape_test.go new file mode 100644 index 000000000..22bb2b04f --- /dev/null +++ b/upstream/registry_fallback_escape_test.go @@ -0,0 +1,194 @@ +package upstream + +import ( + "context" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/health" + "github.com/erpc/erpc/thirdparty" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupRegistryWithFallbacks builds a 4-upstream registry (2 primaries + 2 +// fallbacks) for testing GetFallbackEscapeUpstreams. +func setupRegistryWithFallbacks(t *testing.T, ctx context.Context) (*UpstreamsRegistry, *health.Tracker) { + t.Helper() + util.ResetGock() + util.SetupMocksForEvmStatePoller() + + logger := &log.Logger + metricsTracker := health.NewTracker(logger, "fb-test", 10*time.Second) + metricsTracker.Bootstrap(ctx) + + upstreamConfigs := []*common.UpstreamConfig{ + { + Id: "primary-1", Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + { + Id: "primary-2", Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc2.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + { + Id: "fallback-1", Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc3.localhost", + Tags: []string{common.TagTierFallback}, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + { + Id: "fallback-2", Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc4.localhost", + Tags: []string{common.TagTierFallback}, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + } + + vr := thirdparty.NewVendorsRegistry() + pr, err := thirdparty.NewProvidersRegistry(logger, vr, nil, nil) + require.NoError(t, err) + + ssr, err := data.NewSharedStateRegistry(ctx, logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + }) + require.NoError(t, err) + + registry := NewUpstreamsRegistry(ctx, logger, "fb-test", upstreamConfigs, + ssr, nil, vr, pr, nil, metricsTracker, + nil, + ) + + registry.Bootstrap(ctx) + time.Sleep(100 * time.Millisecond) + require.NoError(t, registry.PrepareUpstreamsForNetwork(ctx, "evm:123")) + return registry, metricsTracker +} + +func findUpstream(t *testing.T, registry *UpstreamsRegistry, id string) *Upstream { + t.Helper() + for _, u := range registry.GetNetworkUpstreams(context.Background(), "evm:123") { + if u.Id() == id { + return u + } + } + t.Fatalf("upstream %q not found", id) + return nil +} + +// Only fallback-tier upstreams are returned. +func TestGetFallbackEscapeUpstreams_FiltersByGroup(t *testing.T) { + defer util.ResetGock() + defer util.AssertNoPendingMocks(t, 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + registry, _ := setupRegistryWithFallbacks(t, ctx) + + out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call") + ids := make([]string, 0, len(out)) + for _, u := range out { + ids = append(ids, u.Id()) + } + assert.ElementsMatch(t, []string{"fallback-1", "fallback-2"}, ids, + "only group=fallback upstreams should be returned; got %v", ids) +} + +// Cordoned fallbacks are still returned: the escape exists to bypass the +// selection policy's cordon. +func TestGetFallbackEscapeUpstreams_IgnoresCordon(t *testing.T) { + defer util.ResetGock() + defer util.AssertNoPendingMocks(t, 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + registry, mt := setupRegistryWithFallbacks(t, ctx) + + fb1 := findUpstream(t, registry, "fallback-1") + fb2 := findUpstream(t, registry, "fallback-2") + + // Simulate the selectionPolicy cordoning both fallbacks. + mt.Cordon(fb1, "*", "excluded by selection policy") + mt.Cordon(fb2, "*", "excluded by selection policy") + require.True(t, mt.IsCordoned(fb1, "eth_call"), "fb1 should be cordoned") + require.True(t, mt.IsCordoned(fb2, "eth_call"), "fb2 should be cordoned") + + out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call") + ids := make([]string, 0, len(out)) + for _, u := range out { + ids = append(ids, u.Id()) + } + assert.ElementsMatch(t, []string{"fallback-1", "fallback-2"}, ids, + "cordoned fallbacks must still be returned for the escape path; got %v", ids) +} + +// Fallbacks that ignore the method are excluded. +func TestGetFallbackEscapeUpstreams_FiltersIgnoreMethods(t *testing.T) { + defer util.ResetGock() + defer util.AssertNoPendingMocks(t, 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + registry, _ := setupRegistryWithFallbacks(t, ctx) + + fb1 := findUpstream(t, registry, "fallback-1") + // Mark eth_call as ignored on fallback-1; AllowMethods left empty. + fb1.config.IgnoreMethods = []string{"eth_call"} + + out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call") + ids := make([]string, 0, len(out)) + for _, u := range out { + ids = append(ids, u.Id()) + } + assert.ElementsMatch(t, []string{"fallback-2"}, ids, + "fallback-1 should be excluded because eth_call is in its IgnoreMethods; got %v", ids) + + // Sanity: a different method that isn't ignored should include fallback-1. + out2 := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_getLogs") + ids2 := make([]string, 0, len(out2)) + for _, u := range out2 { + ids2 = append(ids2, u.Id()) + } + assert.ElementsMatch(t, []string{"fallback-1", "fallback-2"}, ids2, + "eth_getLogs is not ignored; both fallbacks should be returned; got %v", ids2) +} + +// With no fallbacks configured the result is empty. +func TestGetFallbackEscapeUpstreams_NoFallbacksReturnsEmpty(t *testing.T) { + util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Build a registry with primaries only. + logger := &log.Logger + metricsTracker := health.NewTracker(logger, "fb-test", 10*time.Second) + metricsTracker.Bootstrap(ctx) + upstreamConfigs := []*common.UpstreamConfig{ + {Id: "primary-only", Type: common.UpstreamTypeEvm, Endpoint: "http://rpc1.localhost", Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + } + vr := thirdparty.NewVendorsRegistry() + pr, err := thirdparty.NewProvidersRegistry(logger, vr, nil, nil) + require.NoError(t, err) + ssr, err := data.NewSharedStateRegistry(ctx, logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{Driver: "memory", Memory: &common.MemoryConnectorConfig{MaxItems: 1000, MaxTotalSize: "1GB"}}, + }) + require.NoError(t, err) + registry := NewUpstreamsRegistry(ctx, logger, "fb-test", upstreamConfigs, + ssr, nil, vr, pr, nil, metricsTracker, + nil, + ) + registry.Bootstrap(ctx) + time.Sleep(100 * time.Millisecond) + require.NoError(t, registry.PrepareUpstreamsForNetwork(ctx, "evm:123")) + + out := registry.GetFallbackEscapeUpstreams(ctx, "evm:123", "eth_call") + assert.Empty(t, out, "no fallback-group upstreams configured → empty slice") +} diff --git a/upstream/upstream.go b/upstream/upstream.go index 77408b378..251dce30d 100644 --- a/upstream/upstream.go +++ b/upstream/upstream.go @@ -420,6 +420,21 @@ func (u *Upstream) Config() *common.UpstreamConfig { return u.config } +// IsDown reports whether the circuit breaker of the failsafe policy that +// would serve method is open. Breakers scoped to other methods don't count. +func (u *Upstream) IsDown(method string) bool { + if u == nil { + return true + } + req := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest(method, nil)) + fe := u.getFailsafeExecutor(req) + if fe == nil { + return false + } + br := fe.Breaker() + return br != nil && br.State() == failsafe.StateOpen +} + func (u *Upstream) MetricsTracker() *health.Tracker { if u == nil { return nil @@ -634,7 +649,7 @@ func (u *Upstream) Forward(ctx context.Context, nrq *common.NormalizedRequest, b // Send the request based on client type // switch clientType { - case clients.ClientTypeHttpJsonRpc, clients.ClientTypeGrpcBds: + case clients.ClientTypeHttpJsonRpc, clients.ClientTypeGrpcBds, clients.ClientTypeWsJsonRpc: tryForward := func( ctx context.Context, isHedge bool, diff --git a/upstream/upstream_isdown_test.go b/upstream/upstream_isdown_test.go new file mode 100644 index 000000000..14398d43b --- /dev/null +++ b/upstream/upstream_isdown_test.go @@ -0,0 +1,42 @@ +package upstream + +import ( + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/failsafe" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// An open breaker only takes the upstream down for the methods its policy +// serves. +func TestUpstream_IsDown_ScopedToTheMethodsPolicy(t *testing.T) { + lg := zerolog.Nop() + breaker := func(method string) *upstreamExecutor { + ex, err := NewUpstreamExecutor(&common.FailsafeConfig{ + MatchMethod: method, + CircuitBreaker: &common.CircuitBreakerPolicyConfig{ + FailureThresholdCount: 1, + FailureThresholdCapacity: 1, + HalfOpenAfter: common.Duration(time.Hour), + SuccessThresholdCount: 1, + SuccessThresholdCapacity: 1, + }, + }, &lg) + require.NoError(t, err) + return ex + } + traces, catchAll := breaker("trace_*"), breaker("*") + u := &Upstream{failsafeExecutors: []*upstreamExecutor{traces, catchAll}} + + traces.Breaker().Record(failsafe.OutcomeFailure) + require.Equal(t, failsafe.StateOpen, traces.Breaker().State()) + + require.True(t, u.IsDown("trace_block")) + require.False(t, u.IsDown("eth_subscribe"), "a trace_* breaker must not take eth_subscribe down") + + catchAll.Breaker().Record(failsafe.OutcomeFailure) + require.True(t, u.IsDown("eth_subscribe")) +} From 38fec1a4f6d86ac5ae96d31dfb3f8ff66d2380e1 Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 13:28:00 +0100 Subject: [PATCH 04/21] feat(evm): keep latest/finalized reads at the network tip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - enforceHighestBlock also applies to cache hits and keeps the caller's use-upstream selector; when the re-fetch misses it still serves the highest valid block. - Cached latest/finalized entries are keyed by the block the tag currently resolves to, so each tip advance is a new key. - Only the observed tx_replay_attack token is treated as already known. Co-authored-by: shpookas Co-authored-by: João Gomes Co-Authored-By: Claude Opus 5.5 --- architecture/evm/block_ref.go | 51 +++++ architecture/evm/block_ref_test.go | 92 +++++++++ architecture/evm/common.go | 6 +- architecture/evm/error_normalizer.go | 3 +- architecture/evm/error_normalizer_test.go | 47 +++++ architecture/evm/eth_getBlockByNumber.go | 176 +++++++----------- architecture/evm/eth_getBlockByNumber_test.go | 146 +++++++++++++++ architecture/evm/evm_state_poller.go | 12 ++ architecture/evm/evm_state_poller_test.go | 21 +++ architecture/evm/future_block_empty_test.go | 20 +- architecture/evm/json_rpc_cache.go | 8 +- 11 files changed, 460 insertions(+), 122 deletions(-) diff --git a/architecture/evm/block_ref.go b/architecture/evm/block_ref.go index 7c69d075c..a78cc2ed7 100644 --- a/architecture/evm/block_ref.go +++ b/architecture/evm/block_ref.go @@ -114,6 +114,57 @@ func ExtractBlockReferenceFromRequest(ctx context.Context, r *common.NormalizedR return blockRef, blockNumber, nil } +// ResolveCacheBlockRef is ExtractBlockReferenceFromRequest for cache keys: a +// "latest" or "finalized" tag resolves to a concrete block so each tip advance +// gets its own key. It uses the response's block number when given (SET), else +// the network's current tip for the tag (GET). Other refs, including "safe", +// are returned unchanged. +func ResolveCacheBlockRef(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) (string, int64, error) { + blockRef, blockNumber, err := ExtractBlockReferenceFromRequest(ctx, req) + if err != nil { + return blockRef, blockNumber, err + } + + if blockRef != "latest" && blockRef != "finalized" { + return blockRef, blockNumber, nil + } + + if resp != nil { + if _, respBN, rerr := ExtractBlockReferenceFromResponse(ctx, resp); rerr == nil && respBN > 0 { + if hex, herr := common.NormalizeHex(respBN); herr == nil { + return hex, respBN, nil + } + } + } + + // Without a known tip, fall back to the tag-literal key. + net := req.Network() + if net == nil { + return blockRef, blockNumber, nil + } + var num int64 + func() { + defer func() { + if r := recover(); r != nil { + num = 0 + } + }() + switch blockRef { + case "latest": + num = common.EvmHighestLatestBlockNumber(net, ctx) + case "finalized": + num = common.EvmHighestFinalizedBlockNumber(net, ctx) + } + }() + if num > 0 { + if hex, herr := common.NormalizeHex(num); herr == nil { + return hex, num, nil + } + } + + return blockRef, blockNumber, nil +} + func ExtractBlockReferenceFromResponse(ctx context.Context, r *common.NormalizedResponse) (string, int64, error) { ctx, span := common.StartDetailSpan(ctx, "Evm.ExtractBlockReferenceFromResponse") defer span.End() diff --git a/architecture/evm/block_ref_test.go b/architecture/evm/block_ref_test.go index 4dc59079a..48d2dd692 100644 --- a/architecture/evm/block_ref_test.go +++ b/architecture/evm/block_ref_test.go @@ -498,6 +498,98 @@ func TestExtractBlockReference(t *testing.T) { } } +func TestResolveCacheBlockRef(t *testing.T) { + ctx := context.Background() + + t.Run("numeric ref passes through unchanged (no rewrite for non-tag)", func(t *testing.T) { + rpcReq := &common.JsonRpcRequest{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"0x1234", false}, + } + nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq) + + ref, num, err := ResolveCacheBlockRef(ctx, nrq, nil) + assert.NoError(t, err) + assert.Equal(t, "4660", ref) + assert.Equal(t, int64(0x1234), num) + }) + + t.Run("latest tag + response with block number rewrites ref to response hex", func(t *testing.T) { + rpcReq := &common.JsonRpcRequest{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"latest", false}, + } + nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq) + rpcResp := common.MustNewJsonRpcResponseFromBytes(nil, []byte(`{"number":"0xabcdef","hash":"0x1","parentHash":"0x0"}`), nil) + nrs := common.NewNormalizedResponse().WithJsonRpcResponse(rpcResp).WithRequest(nrq) + nrq.SetLastValidResponse(ctx, nrs) + + ref, num, err := ResolveCacheBlockRef(ctx, nrq, nrs) + assert.NoError(t, err) + assert.Equal(t, "0xabcdef", ref, "write path must key by response's actual block number, not 'latest'") + assert.Equal(t, int64(0xabcdef), num) + }) + + t.Run("latest tag with no network and no response falls back to tag literal", func(t *testing.T) { + rpcReq := &common.JsonRpcRequest{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"latest", false}, + } + nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq) + + ref, num, err := ResolveCacheBlockRef(ctx, nrq, nil) + assert.NoError(t, err) + assert.Equal(t, "latest", ref, "with no response and no network, helper must fall back to tag so caller can decide to skip caching") + assert.Equal(t, int64(0), num) + }) + + t.Run("finalized tag rewrite on write path uses response block number", func(t *testing.T) { + rpcReq := &common.JsonRpcRequest{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"finalized", false}, + } + nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq) + rpcResp := common.MustNewJsonRpcResponseFromBytes(nil, []byte(`{"number":"0x100","hash":"0x1","parentHash":"0x0"}`), nil) + nrs := common.NewNormalizedResponse().WithJsonRpcResponse(rpcResp).WithRequest(nrq) + nrq.SetLastValidResponse(ctx, nrs) + + ref, num, err := ResolveCacheBlockRef(ctx, nrq, nrs) + assert.NoError(t, err) + assert.Equal(t, "0x100", ref) + assert.Equal(t, int64(0x100), num) + }) + + t.Run("safe tag keeps the literal key on both paths", func(t *testing.T) { + rpcReq := &common.JsonRpcRequest{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"safe", false}, + } + nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq) + nrq.SetNetwork(&testNetwork{highestLatest: 0x200}) + rpcResp := common.MustNewJsonRpcResponseFromBytes(nil, []byte(`{"number":"0x180","hash":"0x1","parentHash":"0x0"}`), nil) + nrs := common.NewNormalizedResponse().WithJsonRpcResponse(rpcResp).WithRequest(nrq) + + getRef, _, err := ResolveCacheBlockRef(ctx, nrq, nil) + assert.NoError(t, err) + setRef, _, err := ResolveCacheBlockRef(ctx, nrq, nrs) + assert.NoError(t, err) + assert.Equal(t, "safe", getRef) + assert.Equal(t, getRef, setRef, "GET and SET must derive the same key") + }) + + t.Run("earliest tag not rewritten (not tip-bound, existing semantics preserved)", func(t *testing.T) { + rpcReq := &common.JsonRpcRequest{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"earliest", false}, + } + nrq := common.NewNormalizedRequestFromJsonRpcRequest(rpcReq) + + ref, _, err := ResolveCacheBlockRef(ctx, nrq, nil) + assert.NoError(t, err) + assert.Equal(t, "earliest", ref, "earliest does not move with the tip; must not be rewritten") + }) +} + // Operator-defined methods.definitions keys resolve case-insensitively as // well, with an exact-case hit taking precedence (see // common.FindCacheMethodConfig for the tie-break rules). diff --git a/architecture/evm/common.go b/architecture/evm/common.go index 21b152702..3093a6403 100644 --- a/architecture/evm/common.go +++ b/architecture/evm/common.go @@ -29,7 +29,7 @@ func upstreamPostForward_markUnexpectedEmpty( // network's required confidence head (latest by default, or finalized) — it isn't // confirmed yet, so every upstream legitimately returns empty. Return the truthful // empty instead of churning retries until the request times out. - if emptyResultBeyondConfidence(ctx, rq) { + if EmptyResultBeyondConfidence(ctx, rq) { return rs, re } @@ -52,14 +52,14 @@ func upstreamPostForward_markUnexpectedEmpty( ) } -// emptyResultBeyondConfidence reports whether `rq` targets a concrete block number +// EmptyResultBeyondConfidence reports whether `rq` targets a concrete block number // beyond the network's required confidence head — i.e. not yet confirmed enough for an // empty result to mean "missing data" rather than "not produced/finalized yet", so // every upstream legitimately returns empty. The head is the latest head for // EmptyResultConfidence=blockHead (the default) or the finalized head for // finalizedBlock. Returns false (fail-open) when the head is unknown or the request // does not target a concrete numeric block (tags and block-hash lookups never qualify). -func emptyResultBeyondConfidence(ctx context.Context, rq *common.NormalizedRequest) bool { +func EmptyResultBeyondConfidence(ctx context.Context, rq *common.NormalizedRequest) bool { if rq == nil { return false } diff --git a/architecture/evm/error_normalizer.go b/architecture/evm/error_normalizer.go index 8e1366357..d9391ff66 100644 --- a/architecture/evm/error_normalizer.go +++ b/architecture/evm/error_normalizer.go @@ -362,7 +362,8 @@ func ExtractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co strings.Contains(ml, "already in the mempool") || strings.Contains(ml, "transaction already exists") || strings.Contains(ml, "already have transaction") || - strings.Contains(ml, "already exists in mempool") { + strings.Contains(ml, "already exists in mempool") || + strings.Contains(ml, "tx_replay_attack") { // These indicate the exact same transaction is already known - idempotent success case return common.NewErrEndpointNonceException( common.NewErrJsonRpcExceptionInternal( diff --git a/architecture/evm/error_normalizer_test.go b/architecture/evm/error_normalizer_test.go index 1339ba867..69e5ce99a 100644 --- a/architecture/evm/error_normalizer_test.go +++ b/architecture/evm/error_normalizer_test.go @@ -346,6 +346,53 @@ func TestExtractJsonRpcError_MempoolPolicyRejections(t *testing.T) { } } +// A TX_REPLAY_ATTACK rejection of an already-accepted transaction normalizes +// to an "already known" nonce exception; other "replay attack" wording does not. +func TestExtractJsonRpcError_ReplayAttackIdempotency(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + message string + alreadyKnown bool + }{ + { + name: "observed errmsg token", + message: "errcode: 113, errmsg: TX_REPLAY_ATTACK", + alreadyKnown: true, + }, + { + name: "other replay attack wording", + message: "replay attack detected", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + r := &http.Response{StatusCode: 200, Header: http.Header{}} + jrErr := common.NewErrJsonRpcExceptionExternal( + int(common.JsonRpcErrorServerSideException), + tc.message, + "", + ) + jr := common.MustNewJsonRpcResponse(1, nil, jrErr) + + err := ExtractJsonRpcError(r, nil, jr, nil) + if err == nil { + t.Fatalf("expected error, got nil") + } + var ne *common.ErrEndpointNonceException + isAlreadyKnown := errors.As(err, &ne) && + ne.Details["nonceExceptionReason"] == string(common.NonceExceptionReasonAlreadyKnown) + if isAlreadyKnown != tc.alreadyKnown { + t.Fatalf("expected alreadyKnown=%v, got %T: %v", tc.alreadyKnown, err, err) + } + }) + } +} + // TestExtractJsonRpcError_RethRevertError covers reth's Debug-formatted revert, // returned as -32603 with the revert bytes only in the message (seen on HyperEVM // nodes for eth_estimateGas pinned to a block number or "pending"). It is an diff --git a/architecture/evm/eth_getBlockByNumber.go b/architecture/evm/eth_getBlockByNumber.go index ea9e41e96..d034b80a2 100644 --- a/architecture/evm/eth_getBlockByNumber.go +++ b/architecture/evm/eth_getBlockByNumber.go @@ -121,31 +121,28 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common logger := network.Logger().With().Str("method", "eth_getBlockByNumber").Logger() - // If response is from cache, skip enforcement otherwise there's no point in caching. - // As we'll definetely have higher latest block number vs what we have in cache. - // The correct way to deal with this situation is to set proper TTL for "realtime" cache policy. - if nr.FromCache() { - logger.Trace(). - Object("request", nq). - Object("response", nr). - Msg("skipping enforcement of highest block number as response is from cache") - return nr, re - } - + // Cached responses are enforced too: a cached block is served as-is only + // while it still meets the tip. rqj, err := nq.JsonRpcRequest(ctx) if err != nil { return nil, err } rqj.RLock() - defer rqj.RUnlock() - if len(rqj.Params) < 1 { + rqj.RUnlock() return nr, re } bnp, ok := rqj.Params[0].(string) if !ok { + rqj.RUnlock() return nr, re } + var itx bool + if len(rqj.Params) > 1 { + itx, _ = rqj.Params[1].(bool) + } + rqj.RUnlock() + if bnp != "latest" && bnp != "finalized" { return nr, re } @@ -163,19 +160,18 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common if err != nil { return nil, err } - if highestBlockNumber > respBlockNumber { - logger.Debug(). - Str("blockTag", bnp). - Object("request", nq). - Object("response", nr). - Interface("highestBlockNumber", highestBlockNumber). - Interface("respBlockNumber", respBlockNumber). - Interface("err", err). - Msg("enforcing highest latest block") - if respBlockNumber > 0 { - // When extracted block number is 0, it mostly means response is actually a json-rpc error - // therefore we better fetch the highest block number again. - ups := nr.Upstream() + if highestBlockNumber <= respBlockNumber { + return nr, re + } + logger.Debug(). + Str("blockTag", bnp). + Object("request", nq). + Object("response", nr). + Interface("highestBlockNumber", highestBlockNumber). + Interface("respBlockNumber", respBlockNumber). + Msg("enforcing highest latest block") + if respBlockNumber > 0 { + if ups := nr.Upstream(); ups != nil { telemetry.MetricUpstreamStaleLatestBlock.WithLabelValues( network.ProjectId(), ups.VendorName(), @@ -184,57 +180,28 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common "eth_getBlockByNumber", ).Inc() } - var itx bool - if len(rqj.Params) > 1 { - itx, _ = rqj.Params[1].(bool) - } - request, err := BuildGetBlockByNumberRequest(highestBlockNumber, itx) - if err != nil { - return nil, err - } - err = request.SetID(nq.ID()) - if err != nil { - return nil, err - } - newReq := common.NewNormalizedRequestFromJsonRpcRequest(request) - dr := nq.Directives().Clone() - dr.SkipCacheRead = "true" - // In case a block number is extracted, it means the node actually has an older latest block. - // Therefore we exclude the current upstream from the request (as high likely it doesn't have this block). - // Otherwise we still allow the current upstream to be used in case json-rpc error was an intermittent issue. - if respBlockNumber > 0 { - dr.UseUpstream = fmt.Sprintf("!%s", nr.UpstreamId()) - } - newReq.SetDirectives(dr) - newReq.SetNetwork(network) - - // Copy HTTP context (headers, query parameters, user) for proper metrics tracking - newReq.CopyHttpContextFrom(nq) - - nnr, err := network.Forward(ctx, newReq) - // This is needed in case highest block number is corrupted somehow and for example - // it is requesting a very high non-existent block number. - return pickHighestBlock(ctx, nnr, nr, err) - } else { - return nr, re } + + nnr, err := forwardGetBlockByNumber(ctx, network, nq, nr, respBlockNumber, highestBlockNumber, itx) + // This is needed in case highest block number is corrupted somehow and for example + // it is requesting a very high non-existent block number. + return pickHighestBlock(ctx, nnr, nr, err) case "finalized": highestBlockNumber := common.EvmHighestFinalizedBlockNumber(network, tipCtx) _, respBlockNumber, err := ExtractBlockReferenceFromResponse(ctx, nr) if err != nil { return nil, err } - if highestBlockNumber > respBlockNumber { - logger.Debug(). - Str("blockTag", bnp). - Interface("highestBlockNumber", highestBlockNumber). - Interface("respBlockNumber", respBlockNumber). - Interface("err", err). - Msg("enforcing highest finalized block") - if respBlockNumber > 0 { - // When extracted block number is 0, it mostly means response is actually a json-rpc error - // therefore we better fetch the highest block number again. - ups := nr.Upstream() + if highestBlockNumber <= respBlockNumber { + return nr, re + } + logger.Debug(). + Str("blockTag", bnp). + Interface("highestBlockNumber", highestBlockNumber). + Interface("respBlockNumber", respBlockNumber). + Msg("enforcing highest finalized block") + if respBlockNumber > 0 { + if ups := nr.Upstream(); ups != nil { telemetry.MetricUpstreamStaleFinalizedBlock.WithLabelValues( network.ProjectId(), ups.VendorName(), @@ -242,41 +209,9 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common ups.Id(), ).Inc() } - var itx bool - if len(rqj.Params) > 1 { - itx, _ = rqj.Params[1].(bool) - } - request, err := BuildGetBlockByNumberRequest(highestBlockNumber, itx) - if err != nil { - return nil, err - } - err = request.SetID(nq.ID()) - if err != nil { - return nil, err - } - newReq2 := common.NewNormalizedRequestFromJsonRpcRequest(request) - dr := nq.Directives().Clone() - dr.SkipCacheRead = "true" - if respBlockNumber > 0 { - // In case a block number is extracted, it means the node actually has an older latest block. - // Therefore we exclude the current upstream from the request (as high likely it doesn't have this block). - // Otherwise we still allow the current upstream to be used in case json-rpc error was an intermittent issue. - // Also, if response from cache we don't need to exclude the current upstream. - dr.UseUpstream = fmt.Sprintf("!%s", nr.UpstreamId()) - } - newReq2.SetDirectives(dr) - newReq2.SetNetwork(network) - - // Copy HTTP context (headers, query parameters, user) for proper metrics tracking - newReq2.CopyHttpContextFrom(nq) - - nnr, err := network.Forward(ctx, newReq2) - // This is needed in case highest block number is corrupted somehow and for example - // it is requesting a very high non-existent block number. - return pickHighestBlock(ctx, nnr, nr, err) - } else { - return nr, re } + nnr, err := forwardGetBlockByNumber(ctx, network, nq, nr, respBlockNumber, highestBlockNumber, itx) + return pickHighestBlock(ctx, nnr, nr, err) default: return nr, re } @@ -317,7 +252,7 @@ func enforceNonNullBlock(ctx context.Context, nq *common.NormalizedRequest, nr * // isn't produced/confirmed yet and legitimately returns null on every upstream — // it isn't missing/pruned data, so don't convert it to an error and churn retries. // Mirrors the upstream-level markUnexpectedEmpty guard so the two layers agree. - if emptyResultBeyondConfidence(ctx, nq) { + if EmptyResultBeyondConfidence(ctx, nq) { return nr, nil } @@ -338,6 +273,37 @@ func enforceNonNullBlock(ctx context.Context, nq *common.NormalizedRequest, nr * ) } +// forwardGetBlockByNumber re-fetches blockNumber for original. Without a +// caller use-upstream selector, the stale responder is excluded since it +// likely lacks the block. +func forwardGetBlockByNumber( + ctx context.Context, + network common.Network, + original *common.NormalizedRequest, + stale *common.NormalizedResponse, + staleBlockNumber int64, + blockNumber int64, + includeTx bool, +) (*common.NormalizedResponse, error) { + request, err := BuildGetBlockByNumberRequest(blockNumber, includeTx) + if err != nil { + return nil, err + } + if err := request.SetID(original.ID()); err != nil { + return nil, err + } + newReq := common.NewNormalizedRequestFromJsonRpcRequest(request) + dr := original.Directives().Clone() + dr.SkipCacheRead = "true" + if dr.UseUpstream == "" && staleBlockNumber > 0 && !stale.FromCache() { + dr.UseUpstream = fmt.Sprintf("!%s", stale.UpstreamId()) + } + newReq.SetDirectives(dr) + newReq.SetNetwork(network) + newReq.CopyHttpContextFrom(original) + return network.Forward(ctx, newReq) +} + func pickHighestBlock(ctx context.Context, x *common.NormalizedResponse, y *common.NormalizedResponse, err error) (*common.NormalizedResponse, error) { ctx, span := common.StartDetailSpan(ctx, "Evm.PickHighestBlock") defer span.End() diff --git a/architecture/evm/eth_getBlockByNumber_test.go b/architecture/evm/eth_getBlockByNumber_test.go index 987fce12d..17e6032ac 100644 --- a/architecture/evm/eth_getBlockByNumber_test.go +++ b/architecture/evm/eth_getBlockByNumber_test.go @@ -7,6 +7,7 @@ import ( "github.com/erpc/erpc/common" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // testNetwork is a simple test implementation of common.Network interface for this file @@ -194,3 +195,148 @@ func TestEnforceNonNullTaggedBlocks(t *testing.T) { assert.True(t, result.IsResultEmptyish()) }) } + +// tipFetchNetwork serves any concrete block number from Forward so the tip +// re-fetch in enforceHighestBlock can succeed. +type tipFetchNetwork struct { + testNetwork + finalizedTip int64 +} + +func (n *tipFetchNetwork) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 { + return n.finalizedTip +} + +func (n *tipFetchNetwork) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + rqj, err := req.JsonRpcRequest(ctx) + if err != nil { + return nil, err + } + jrr, err := common.NewJsonRpcResponse(rqj.ID, map[string]interface{}{"number": rqj.Params[0]}, nil) + if err != nil { + return nil, err + } + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr), nil +} + +// A cached response carries no upstream. When it lags the tip, enforcement +// must re-fetch the tip instead of dereferencing the missing upstream. +func TestEnforceHighestBlock_CachedResponseWithoutUpstream(t *testing.T) { + cases := []struct { + tag string + latestTip int64 + finalizedTip int64 + cachedNumber string + expectedBlock int64 + }{ + {tag: "latest", latestTip: 0x101, cachedNumber: "0x100", expectedBlock: 0x101}, + {tag: "finalized", latestTip: 0x200, finalizedTip: 0x101, cachedNumber: "0x100", expectedBlock: 0x101}, + } + + for _, tc := range cases { + t.Run(tc.tag, func(t *testing.T) { + network := &tipFetchNetwork{ + testNetwork: testNetwork{highestLatest: tc.latestTip}, + finalizedTip: tc.finalizedTip, + } + req := common.NewNormalizedRequestFromJsonRpcRequest( + common.NewJsonRpcRequest("eth_getBlockByNumber", []interface{}{tc.tag, false}), + ) + req.SetDirectives(&common.RequestDirectives{EnforceHighestBlock: true}) + jrr, err := common.NewJsonRpcResponse(1, map[string]interface{}{"number": tc.cachedNumber}, nil) + assert.NoError(t, err) + cached := common.NewNormalizedResponse().WithRequest(req).WithFromCache(true).WithJsonRpcResponse(jrr) + assert.Nil(t, cached.Upstream()) + + var out *common.NormalizedResponse + assert.NotPanics(t, func() { + out, err = enforceHighestBlock(context.Background(), network, req, cached, nil) + }) + assert.NoError(t, err) + assert.NotNil(t, out) + _, bn, err := ExtractBlockReferenceFromResponse(context.Background(), out) + assert.NoError(t, err) + assert.Equal(t, tc.expectedBlock, bn) + }) + } +} + +// refetchNetwork scopes the latest tip to the request's use-upstream selector +// (as Network does), answers every tip re-fetch with null and records the +// selector each re-fetch was forwarded with. +type refetchNetwork struct { + testNetwork + scopedTip int64 + forwarded []string +} + +func (n *refetchNetwork) EvmHighestLatestBlockNumber(ctx context.Context) int64 { + if req, ok := ctx.Value(common.RequestContextKey).(*common.NormalizedRequest); ok && req.Directives().UseUpstream != "" { + return n.scopedTip + } + return n.highestLatest +} + +func (n *refetchNetwork) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + n.forwarded = append(n.forwarded, req.Directives().UseUpstream) + jrr, err := common.NewJsonRpcResponse(1, nil, nil) + if err != nil { + return nil, err + } + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr), nil +} + +func latestBlockResponse(t *testing.T, useUpstream string, number string, fromCache bool) (*common.NormalizedRequest, *common.NormalizedResponse) { + req := common.NewNormalizedRequestFromJsonRpcRequest( + common.NewJsonRpcRequest("eth_getBlockByNumber", []interface{}{"latest", false}), + ) + req.SetDirectives(&common.RequestDirectives{EnforceHighestBlock: true, UseUpstream: useUpstream}) + jrr, err := common.NewJsonRpcResponse(1, map[string]interface{}{"number": number, "hash": "0x01"}, nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithRequest(req).WithFromCache(fromCache).WithJsonRpcResponse(jrr) + if !fromCache { + resp.SetUpstream(common.NewFakeUpstream("rpc1")) + } + return req, resp +} + +// When the tip re-fetch misses, the stale-but-valid block is served rather +// than an error, after a single re-fetch. +func TestEnforceHighestBlock_LatestRefetchMissFailsOpen(t *testing.T) { + for _, fromCache := range []bool{false, true} { + network := &refetchNetwork{testNetwork: testNetwork{highestLatest: 100}} + req, resp := latestBlockResponse(t, "", "0x63", fromCache) + + out, err := enforceHighestBlock(context.Background(), network, req, resp, nil) + require.NoError(t, err) + _, bn, err := ExtractBlockReferenceFromResponse(context.Background(), out) + require.NoError(t, err) + assert.Equal(t, int64(99), bn) + + // The stale responder is excluded; a cache hit has none to exclude. + want := "!rpc1" + if fromCache { + want = "" + } + assert.Equal(t, []string{want}, network.forwarded, "fromCache=%v", fromCache) + } +} + +// A use-upstream-scoped request is enforced against its group's tip, and the +// re-fetch keeps the caller's selector. +func TestEnforceHighestBlock_KeepsUseUpstreamScope(t *testing.T) { + for _, fromCache := range []bool{false, true} { + network := &refetchNetwork{testNetwork: testNetwork{highestLatest: 2000}, scopedTip: 1000} + + req, resp := latestBlockResponse(t, "slow*", "0x3e8", fromCache) + out, err := enforceHighestBlock(context.Background(), network, req, resp, nil) + require.NoError(t, err) + assert.Same(t, resp, out) + assert.Empty(t, network.forwarded, "fromCache=%v: at the group tip", fromCache) + + req, resp = latestBlockResponse(t, "slow*", "0x3e7", fromCache) + _, err = enforceHighestBlock(context.Background(), network, req, resp, nil) + require.NoError(t, err) + assert.Equal(t, []string{"slow*"}, network.forwarded, "fromCache=%v", fromCache) + } +} diff --git a/architecture/evm/evm_state_poller.go b/architecture/evm/evm_state_poller.go index 470f95750..9bffe4475 100644 --- a/architecture/evm/evm_state_poller.go +++ b/architecture/evm/evm_state_poller.go @@ -502,6 +502,13 @@ func (e *EvmStatePoller) PollLatestBlockNumber(ctx context.Context) (int64, erro e.stateMu.Unlock() return 0, nil } else { + // An open circuit breaker short-circuits before tryForward + // records the failure, which would freeze the upstream's error + // rate; record it here. Other failures are already recorded. + if e.tracker != nil && common.HasErrorCode(err, common.ErrCodeFailsafeCircuitBreakerOpen) { + e.tracker.RecordUpstreamRequest(e.upstream, "eth_getBlockByNumber", common.DataFinalityStateRealtime) + e.tracker.RecordUpstreamFailure(e.upstream, "eth_getBlockByNumber", common.DataFinalityStateRealtime, err) + } e.logger.Warn().Err(err).Msg("failed to get latest block number in evm state poller") return 0, err } @@ -794,6 +801,11 @@ func (e *EvmStatePoller) PollFinalizedBlockNumber(ctx context.Context) (int64, e e.stateMu.Unlock() return 0, nil } else { + // See PollLatestBlockNumber. + if e.tracker != nil && common.HasErrorCode(err, common.ErrCodeFailsafeCircuitBreakerOpen) { + e.tracker.RecordUpstreamRequest(e.upstream, "eth_getBlockByNumber", common.DataFinalityStateFinalized) + e.tracker.RecordUpstreamFailure(e.upstream, "eth_getBlockByNumber", common.DataFinalityStateFinalized, err) + } e.logger.Warn().Err(err).Msg("failed to get finalized block number in evm state poller") return 0, err } diff --git a/architecture/evm/evm_state_poller_test.go b/architecture/evm/evm_state_poller_test.go index 27fedff35..c1ee1f68e 100644 --- a/architecture/evm/evm_state_poller_test.go +++ b/architecture/evm/evm_state_poller_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "runtime" + "sync" "sync/atomic" "testing" "time" @@ -173,3 +174,23 @@ func TestEvmStatePoller_BootstrapWithZeroIntervalStartsNothing(t *testing.T) { assert.Equal(t, int64(0), up.forwards.Load(), "interval=0 must not poll the upstream") assert.False(t, poller.Enabled) } + +// Request-path polls (block-availability gates, block-skip refresh) respect +// the debounce: concurrent and repeated calls within it share one fetch. +func TestEvmStatePoller_RequestPathPollsAreCoalesced(t *testing.T) { + poller, up, ctx := newTestStatePoller(t, time.Hour, time.Minute) + + var wg sync.WaitGroup + for range 20 { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = poller.PollLatestBlockNumber(ctx) + }() + } + wg.Wait() + for range 20 { + _, _ = poller.PollLatestBlockNumber(ctx) + } + assert.Equal(t, int64(1), up.forwards.Load()) +} diff --git a/architecture/evm/future_block_empty_test.go b/architecture/evm/future_block_empty_test.go index 554a1978d..a232b5161 100644 --- a/architecture/evm/future_block_empty_test.go +++ b/architecture/evm/future_block_empty_test.go @@ -26,15 +26,15 @@ func TestEmptyResultBeyondConfidence(t *testing.T) { return req } - assert.False(t, emptyResultBeyondConfidence(ctx, mk(nw, 999)), "behind head → retryable, not beyond") - assert.False(t, emptyResultBeyondConfidence(ctx, mk(nw, 1000)), "exactly head → not beyond") - assert.True(t, emptyResultBeyondConfidence(ctx, mk(nw, 1001)), "head+1 → beyond, return empty") - assert.True(t, emptyResultBeyondConfidence(ctx, mk(nw, 9_000_000)), "far ahead → beyond") - assert.False(t, emptyResultBeyondConfidence(ctx, mk(nw, 0)), "no concrete block (tag/hash) → never beyond") + assert.False(t, EmptyResultBeyondConfidence(ctx, mk(nw, 999)), "behind head → retryable, not beyond") + assert.False(t, EmptyResultBeyondConfidence(ctx, mk(nw, 1000)), "exactly head → not beyond") + assert.True(t, EmptyResultBeyondConfidence(ctx, mk(nw, 1001)), "head+1 → beyond, return empty") + assert.True(t, EmptyResultBeyondConfidence(ctx, mk(nw, 9_000_000)), "far ahead → beyond") + assert.False(t, EmptyResultBeyondConfidence(ctx, mk(nw, 0)), "no concrete block (tag/hash) → never beyond") // Unknown head (0) → fail open. noHead := &queryTestNetwork{cfg: nw.cfg, latest: 0} - assert.False(t, emptyResultBeyondConfidence(ctx, mk(noHead, 9_000_000)), "unknown head → fail open") + assert.False(t, EmptyResultBeyondConfidence(ctx, mk(noHead, 9_000_000)), "unknown head → fail open") } // TestEmptyResultBeyondConfidence_Finalized pins the finalizedBlock confidence level: @@ -58,10 +58,10 @@ func TestEmptyResultBeyondConfidence_Finalized(t *testing.T) { return req } - assert.False(t, emptyResultBeyondConfidence(ctx, mk(899)), "below finalized → retryable") - assert.False(t, emptyResultBeyondConfidence(ctx, mk(900)), "exactly finalized → not beyond") - assert.True(t, emptyResultBeyondConfidence(ctx, mk(901)), "finalized+1 (unfinalized) → beyond at finalized confidence") - assert.True(t, emptyResultBeyondConfidence(ctx, mk(1000)), "latest head but unfinalized → beyond at finalized confidence") + assert.False(t, EmptyResultBeyondConfidence(ctx, mk(899)), "below finalized → retryable") + assert.False(t, EmptyResultBeyondConfidence(ctx, mk(900)), "exactly finalized → not beyond") + assert.True(t, EmptyResultBeyondConfidence(ctx, mk(901)), "finalized+1 (unfinalized) → beyond at finalized confidence") + assert.True(t, EmptyResultBeyondConfidence(ctx, mk(1000)), "latest head but unfinalized → beyond at finalized confidence") } // TestEnforceNonNullBlock_FutureBlockNotErrored pins the #6 fix: the network-level diff --git a/architecture/evm/json_rpc_cache.go b/architecture/evm/json_rpc_cache.go index c48cf12b4..7eaa09f58 100644 --- a/architecture/evm/json_rpc_cache.go +++ b/architecture/evm/json_rpc_cache.go @@ -666,7 +666,8 @@ func (c *EvmJsonRpcCache) Set(ctx context.Context, req *common.NormalizedRequest attribute.String("network.id", ntwId), ) - blockRef, blockNumber, err := ExtractBlockReferenceFromRequest(ctx, req) + // Key "latest"/"finalized" by the response's block (see ResolveCacheBlockRef). + blockRef, blockNumber, err := ResolveCacheBlockRef(ctx, req, resp) if err != nil { common.SetTraceSpanError(span, err) return err @@ -1086,7 +1087,8 @@ func (c *EvmJsonRpcCache) doGet(ctx context.Context, connector data.Connector, r rpcReq.RLockWithTrace(ctx) defer rpcReq.RUnlock() - blockRef, _, err := ExtractBlockReferenceFromRequest(ctx, req) + // Key "latest"/"finalized" by the network's current tip (see ResolveCacheBlockRef). + blockRef, _, err := ResolveCacheBlockRef(ctx, req, nil) if err != nil { return nil, err } @@ -1208,7 +1210,7 @@ func shouldCacheResponse( // Never cache an empty result for a not-yet-produced (future) block: the block // will exist later, so a cached null would be served as a wrong answer until the // TTL expires. This holds regardless of the policy's empty behavior. - if isEmpty && resp != nil && emptyResultBeyondConfidence(ctx, resp.Request()) { + if isEmpty && resp != nil && EmptyResultBeyondConfidence(ctx, resp.Request()) { lg.Debug().Msg("skip caching empty result for a not-yet-produced (future) block") return false, nil } From 5fa07209e75c5b50b87fa2981c37722af66b837a Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 13:28:24 +0100 Subject: [PATCH 05/21] feat(network): delivered-head floor and per-request fallback escape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delivered-head floor: a head already delivered on a WebSocket subscription, from an upstream eligible to serve the network, floors "latest" so HTTP never regresses below it. It is bounded against live heads, per project, skipped for use-upstream requests, never lifts past servedTip guarantees, and shared across instances in the background. - failover.onDefaultsExhausted: when every routed upstream failed with an error retryable toward the network, retry the request once on tier:fallback upstreams not yet tried, before the selection policy promotes them. Empty results count only for a block that should exist; consensus is excluded. The escape honours a use-upstream pin, and keeps the routed upstreams' empty answer if every fallback fails. - Near-tip eth_getBlockByNumber prefers an upstream strictly ahead of the others; requests for a specific block try upstreams that have it first, preserving score order. - Non-retryable writes are not multiplexed. Co-authored-by: shpookas Co-authored-by: João Gomes Co-Authored-By: Claude Opus 5.5 --- erpc/networks.go | 574 +++++++++---- erpc/networks_block_partition_test.go | 105 +++ erpc/networks_consensus_test.go | 1 + erpc/networks_earliest_detection_test.go | 3 +- erpc/networks_failover_escape_test.go | 780 ++++++++++++++++++ erpc/networks_finality_interpolation_test.go | 65 ++ erpc/networks_multiplexer_test.go | 19 + erpc/networks_registry.go | 11 + .../networks_served_tip_guaranteedfor_test.go | 30 + erpc/networks_test.go | 50 ++ erpc/networks_tip_floor_test.go | 72 ++ 11 files changed, 1555 insertions(+), 155 deletions(-) create mode 100644 erpc/networks_block_partition_test.go create mode 100644 erpc/networks_failover_escape_test.go create mode 100644 erpc/networks_tip_floor_test.go diff --git a/erpc/networks.go b/erpc/networks.go index 7d43a6610..bbd2a509d 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -17,6 +17,7 @@ import ( "github.com/erpc/erpc/architecture/evm" "github.com/erpc/erpc/architecture/svm" "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" "github.com/erpc/erpc/health" "github.com/erpc/erpc/internal/policy" "github.com/erpc/erpc/telemetry" @@ -51,6 +52,14 @@ type Network struct { initializer *util.Initializer architectureHandler common.ArchitectureHandler + // latestBlockShared is the highest head any instance has delivered to a + // WebSocket subscriber (see applyDeliveredHeadFloor). Nil when shared + // state is not configured. + latestBlockShared data.CounterInt64SharedVariable + // deliveredLatestBlock is the highest head this instance has delivered to + // a WebSocket subscriber. + deliveredLatestBlock atomic.Int64 + // servedLatest / servedFinalized are STRICT-MONOTONIC at the network level: // once we serve a tip of N to clients, EvmHighestLatest/FinalizedBlockNumber // servedTipAnchor watchdogs track when this process last SAW the served @@ -79,6 +88,33 @@ type Network struct { servedTipBlockTimeOverride float64 } +// NoteObservedLatestBlock records a head about to be delivered on a WebSocket +// newHeads subscription, so "latest" over HTTP never trails it. Callers must +// invoke it before delivering the notification. +func (n *Network) NoteObservedLatestBlock(ctx context.Context, blockNumber int64) { + if n == nil || blockNumber <= 0 { + return + } + if ctx == nil { + ctx = n.appCtx + } + if ctx == nil { + ctx = context.Background() + } + if n.latestBlockShared != nil { + n.latestBlockShared.TryUpdate(ctx, blockNumber) + } + for { + cur := n.deliveredLatestBlock.Load() + if blockNumber <= cur { + return + } + if n.deliveredLatestBlock.CompareAndSwap(cur, blockNumber) { + return + } + } +} + // maxServedTipPartitions caps the number of materialized per-tag served-tip // partitions per network (a backstop on top of the "must be a configured tag" // gate). Realistic configs have 1–3 groups; beyond the cap, extra tags fall @@ -794,8 +830,13 @@ func (n *Network) EvmHighestLatestBlockNumber(ctx context.Context) int64 { if !n.servedTipEnabledFor("latest") { // tipCandidateUpstreams already scopes to the request's selector (if - // any), so the head is within-subset. - return n.evmHeadReference(ctx, false).Corroborated + // any), so the head is within-subset. The delivered-head floor is + // network-wide, so it applies to unscoped requests only. + ref := n.evmHeadReference(ctx, false) + if requestSelector(ctx) != "" { + return ref.Corroborated + } + return n.applyDeliveredHeadFloor(ref.Corroborated, ref.Max) } if sel := requestSelector(ctx); sel != "" { // Targeted request: the gather is already scoped to the selector's @@ -807,7 +848,16 @@ func (n *Network) EvmHighestLatestBlockNumber(ctx context.Context) int64 { } return n.servedTip(ctx, span, false, "latest", nil, servedTipLaneNone) } - return n.servedTip(ctx, span, false, "latest", &n.servedLatestAnchor, "") + pick := n.servedTip(ctx, span, false, "latest", &n.servedLatestAnchor, "") + floored := n.applyDeliveredHeadFloor(pick, n.evmHeadReference(ctx, false).Max) + if floored > pick { + // The floor may lift past the majority but never past a guarantee + // (servedTip already clamped pick to it). + if g := n.guaranteedFloor(ctx, false); g > 0 && g < floored { + floored = max(pick, g) + } + } + return floored } // servedTipEnabledFor reports whether the majority served tip is enabled for @@ -1504,6 +1554,36 @@ func (n *Network) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 { return n.servedTip(ctx, span, true, "finalized", &n.servedFinalizedAnchor, "") } +// applyDeliveredHeadFloor lifts the computed network-wide "latest" head to the +// highest head already delivered to a WebSocket subscriber by this or any +// instance, so a client never sees "latest" regress below a streamed head. +// Only delivered heads feed the floor; remembering the computed head would +// turn the served tip into a persisted clamp. +// +// A delivered head always came from some live upstream, so the floor is +// honoured only within maxRetryableBlockDistance of the freshest live head +// (liveMax), or when no head is known at all. A stale shared value is ignored +// and ages out as the chain advances. +func (n *Network) applyDeliveredHeadFloor(computed, liveMax int64) int64 { + floor := n.deliveredLatestBlock.Load() + if n.latestBlockShared != nil { + floor = max(floor, n.latestBlockShared.GetValue()) + } + if floor <= computed { + return computed + } + ceiling := max(liveMax, computed) + if ceiling <= 0 || floor-ceiling <= n.maxRetryableBlockDistance() { + return floor + } + n.logger.Debug(). + Int64("computed", computed). + Int64("liveMax", liveMax). + Int64("deliveredFloor", floor). + Msg("ignoring delivered-head floor too far ahead of every live head") + return computed +} + // guaranteedFloor returns the lowest majority served tip across every // configured guarantee SUBSET, or 0 when none is configured or none constrains // the tip. Two kinds of subset, one rule: @@ -1972,6 +2052,21 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* return nil, err } + // For a specific block, try upstreams whose poller already has it first. + // Ordering hints only: every upstream stays eligible. + if n.Architecture() == common.ArchitectureEvm { + if bn := requestBlockNumber(ctx, req); bn > 0 { + upsList = partitionUpstreamsByLatestBlock(upsList, bn) + upsList = preferTipLeaderForNearTipGetBlock(upsList, method, bn) + } + } + + // Failover tiering: try default-tier upstreams before fallback-tier ones, + // keeping score order within each tier. + if n.cfg.Failover.Enabled() { + upsList = tierUpstreamsByGroup(upsList) + } + // Architecture-specific pruning of the upstream list. Currently only SVM // uses this hook; both filters are gated on ArchitectureSvm so EVM networks // never enter this block. @@ -2200,178 +2295,252 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* maxLoopIterations = 1 } attempted := make(map[string]struct{}, maxLoopIterations) - - for loopIteration := 0; loopIteration < maxLoopIterations; loopIteration++ { - loopCtx, loopSpan := common.StartDetailSpan(execSpanCtx, "Network.UpstreamLoop") - if loopIteration > 0 { - // Non-first picks of this execution are sweep iterations — - // tag the context so the attempt record reads sweep instead - // of primary/retry (see common.WithSweepIteration). - loopCtx = common.WithSweepIteration(loopCtx) - } - if ctxErr := loopCtx.Err(); ctxErr != nil { - cause := context.Cause(loopCtx) - if cause == nil { - cause = ctxErr + // The fallback escape below swaps in its own picker, leaving the + // request's routed list untouched for later retries. + nextUpstream := effectiveReq.NextUpstream + // An empty result is a miss (rather than a legitimate answer, like a + // pending tx's null receipt) only for a concrete block the network + // is confident about. + emptyIsMiss := func(ctx context.Context) bool { + return requestBlockNumber(ctx, effectiveReq) > 0 && !evm.EmptyResultBeyondConfidence(ctx, effectiveReq) + } + + escalationLoop: + for { + for loopIteration := 0; loopIteration < maxLoopIterations; loopIteration++ { + loopCtx, loopSpan := common.StartDetailSpan(execSpanCtx, "Network.UpstreamLoop") + if loopIteration > 0 { + // Non-first picks of this execution are sweep iterations — + // tag the context so the attempt record reads sweep instead + // of primary/retry (see common.WithSweepIteration). + loopCtx = common.WithSweepIteration(loopCtx) + } + if ctxErr := loopCtx.Err(); ctxErr != nil { + cause := context.Cause(loopCtx) + if cause == nil { + cause = ctxErr + } + common.SetTraceSpanError(loopSpan, cause) + loopSpan.End() + return nil, cause } - common.SetTraceSpanError(loopSpan, cause) - loopSpan.End() - return nil, cause - } - - u, selErr := effectiveReq.NextUpstream() - if selErr != nil { - loopSpan.SetAttributes( - attribute.Bool("upstreams_exhausted", true), - attribute.String("error", selErr.Error()), - ) - loopSpan.End() - break - } - - if _, seen := attempted[u.Id()]; seen { - // Already tried in this execution — MarkUpstreamCompleted freed - // it from ConsumedUpstreams (retryable error or empty result) and - // UpstreamIdx wrapped around. Release the reservation so the - // upstream is available for the next failsafe retry round. - effectiveReq.ConsumedUpstreams.Delete(u) - loopSpan.SetAttributes(attribute.Bool("duplicate_selection", true)) - loopSpan.End() - break - } - attempted[u.Id()] = struct{}{} - loopSpan.SetAttributes(attribute.String("upstream.id", u.Id())) - if eu, ok := u.(common.EvmUpstream); ok { - if sp := eu.EvmStatePoller(); sp != nil && !sp.IsObjectNull() { + u, selErr := nextUpstream() + if selErr != nil { loopSpan.SetAttributes( - attribute.Int64("upstream.latest_block", sp.LatestBlock()), - attribute.Int64("upstream.finalized_block", sp.FinalizedBlock()), + attribute.Bool("upstreams_exhausted", true), + attribute.String("error", selErr.Error()), ) + loopSpan.End() + break } - } - ulg := lg.With().Str("upstreamId", u.Id()).Logger() - ulg.Debug(). - Interface("id", effectiveReq.ID()). - Str("ptr", fmt.Sprintf("%p", effectiveReq)). - Str("selectedUpstream", u.Id()). - Msg("selected upstream from list") + if _, seen := attempted[u.Id()]; seen { + // Already tried in this execution — MarkUpstreamCompleted freed + // it from ConsumedUpstreams (retryable error or empty result) and + // UpstreamIdx wrapped around. Release the reservation so the + // upstream is available for the next failsafe retry round. + effectiveReq.ConsumedUpstreams.Delete(u) + loopSpan.SetAttributes(attribute.Bool("duplicate_selection", true)) + loopSpan.End() + break + } + attempted[u.Id()] = struct{}{} + + loopSpan.SetAttributes(attribute.String("upstream.id", u.Id())) + if eu, ok := u.(common.EvmUpstream); ok { + if sp := eu.EvmStatePoller(); sp != nil && !sp.IsObjectNull() { + loopSpan.SetAttributes( + attribute.Int64("upstream.latest_block", sp.LatestBlock()), + attribute.Int64("upstream.finalized_block", sp.FinalizedBlock()), + ) + } + } - // Pre-forward: block availability gating → skip to next upstream - if skipErr, isRetryable := n.checkUpstreamBlockAvailability(loopCtx, u, effectiveReq, method); skipErr != nil { - n.handleBlockSkip(loopCtx, loopSpan, &ulg, u, effectiveReq, method, skipErr, isRetryable) - loopSpan.End() - continue - } + ulg := lg.With().Str("upstreamId", u.Id()).Logger() + ulg.Debug(). + Interface("id", effectiveReq.ID()). + Str("ptr", fmt.Sprintf("%p", effectiveReq)). + Str("selectedUpstream", u.Id()). + Msg("selected upstream from list") + + // Pre-forward: block availability gating → skip to next upstream + if skipErr, isRetryable := n.checkUpstreamBlockAvailability(loopCtx, u, effectiveReq, method); skipErr != nil { + n.handleBlockSkip(loopCtx, loopSpan, &ulg, u, effectiveReq, method, skipErr, isRetryable) + // Lets the fallback escape below see the skip; a consensus + // slot keeps reporting it as no attempt. + if !oneUpstreamOnly { + lastErr = skipErr + } + loopSpan.End() + continue + } - hedges := snap.Hedges - attempts := snap.Attempts - if hedges > 0 { - finality := effectiveReq.Finality(loopCtx) - telemetry.CounterHandle(telemetry.MetricNetworkHedgedRequestTotal, - n.projectId, n.Label(), u.Id(), method, fmt.Sprintf("%d", hedges), - finality.String(), effectiveReq.UserId(), effectiveReq.AgentName(), - ).Inc() - } + hedges := snap.Hedges + attempts := snap.Attempts + if hedges > 0 { + finality := effectiveReq.Finality(loopCtx) + telemetry.CounterHandle(telemetry.MetricNetworkHedgedRequestTotal, + n.projectId, n.Label(), u.Id(), method, fmt.Sprintf("%d", hedges), + finality.String(), effectiveReq.UserId(), effectiveReq.AgentName(), + ).Inc() + } - r, err := tryForward(u, effectiveReq, loopCtx, &ulg, hedges, attempts, snap.Retries) - if e := n.normalizeResponse(loopCtx, effectiveReq, r); e != nil { - ulg.Error().Err(e).Msgf("failed to normalize response") - err = e - } - effectiveReq.MarkUpstreamCompleted(loopCtx, u, r, err) + r, err := tryForward(u, effectiveReq, loopCtx, &ulg, hedges, attempts, snap.Retries) + if e := n.normalizeResponse(loopCtx, effectiveReq, r); e != nil { + ulg.Error().Err(e).Msgf("failed to normalize response") + err = e + } + effectiveReq.MarkUpstreamCompleted(loopCtx, u, r, err) - // Hedge cancelled → this execution lost the race, bail out - if hedges > 0 && common.HasErrorCode(err, common.ErrCodeEndpointRequestCanceled) { - n.recordHedgeDiscard(loopCtx, loopSpan, &ulg, u, effectiveReq, method, err, attempts, hedges) - loopSpan.End() - return nil, common.NewErrUpstreamHedgeCancelled(u.Id(), err) - } - _ = attempts // keep symbol live for future telemetry callsites + // Hedge cancelled → this execution lost the race, bail out + if hedges > 0 && common.HasErrorCode(err, common.ErrCodeEndpointRequestCanceled) { + n.recordHedgeDiscard(loopCtx, loopSpan, &ulg, u, effectiveReq, method, err, attempts, hedges) + loopSpan.End() + return nil, common.NewErrUpstreamHedgeCancelled(u.Id(), err) + } + _ = attempts // keep symbol live for future telemetry callsites - if r != nil { - r.SetUpstream(u) - r.WithRequest(effectiveReq) - } + if r != nil { + r.SetUpstream(u) + r.WithRequest(effectiveReq) + } - // Return immediately when the result is usable: - // - Non-empty success always qualifies. - // - Emptyish success qualifies when the method is in - // emptyResultAccept and consensus is not required, - // because failsafe would accept the empty result anyway - // so trying more upstreams just wastes time on slow ones. - // - Otherwise emptyish results continue to the next upstream. - if err == nil && r != nil && !r.IsObjectNull() { - emptyish := isEmptyNetworkResult(method, r) - acceptEmpty := !emptyish || - (!failsafeExecutor.HasConsensus() && - failsafeExecutor.acceptsEmptyResult(method, r)) - if acceptEmpty { - st := effectiveReq.ExecState() - st.MarkUpstreamAttemptWon(r.UpstreamId()) - s := st.Snapshot() - r.SetAttempts(s.Attempts) - r.SetRetries(s.Retries) - r.SetHedges(s.Hedges) - loopSpan.SetStatus(codes.Ok, "") - if emptyish { - loopSpan.SetAttributes(attribute.Bool("emptyish_accepted", true)) + // Return immediately when the result is usable: + // - Non-empty success always qualifies. + // - Emptyish success qualifies when the method is in + // emptyResultAccept and consensus is not required, + // because failsafe would accept the empty result anyway + // so trying more upstreams just wastes time on slow ones. + // - Otherwise emptyish results continue to the next upstream. + if err == nil && r != nil && !r.IsObjectNull() { + emptyish := isEmptyNetworkResult(method, r) + acceptEmpty := !emptyish || + (!failsafeExecutor.HasConsensus() && + failsafeExecutor.acceptsEmptyResult(method, r)) + if acceptEmpty { + st := effectiveReq.ExecState() + st.MarkUpstreamAttemptWon(r.UpstreamId()) + s := st.Snapshot() + r.SetAttempts(s.Attempts) + r.SetRetries(s.Retries) + r.SetHedges(s.Hedges) + loopSpan.SetStatus(codes.Ok, "") + if emptyish { + loopSpan.SetAttributes(attribute.Bool("emptyish_accepted", true)) + } + loopSpan.End() + return r, nil } + } + + // Deterministic errors: client faults and execution reverts are the + // same on every upstream — no point trying others. + if common.IsClientError(err) || common.HasErrorCode(err, common.ErrCodeEndpointExecutionException) { + common.SetTraceSpanError(loopSpan, err) loopSpan.End() - return r, nil + return nil, err } - } - // Deterministic errors: client faults and execution reverts are the - // same on every upstream — no point trying others. - if common.IsClientError(err) || common.HasErrorCode(err, common.ErrCodeEndpointExecutionException) { - common.SetTraceSpanError(loopSpan, err) + // Track best result and continue to next upstream. + if err != nil { + lastErr = err + common.SetTraceSpanError(loopSpan, err) + } else if r != nil { + bestResp = r + loopSpan.SetStatus(codes.Ok, "") + if isEmptyNetworkResult(method, r) && n.cfg.Failover.Enabled() && emptyIsMiss(loopCtx) { + lastErr = common.NewErrEndpointMissingData( + fmt.Errorf("upstream responded emptyish"), u, + ) + } + } loopSpan.End() - return nil, err - } - // Track best result and continue to next upstream. - if err != nil { - lastErr = err - common.SetTraceSpanError(loopSpan, err) - } else if r != nil { - bestResp = r - loopSpan.SetStatus(codes.Ok, "") + // An error explicitly marked non-retryable toward the network is + // an authoritative answer — e.g. an OP-Stack sequencer rate limit + // every provider shares. Every remaining upstream returns the same + // verdict, so sweeping them burns latency and quota for nothing. + // Same stop condition shouldRetryWithReason already applies on the + // retry path. + // + // A skipped slot is deliberately NOT an example here. Solana's + // -32009 reads "skipped, OR missing in long-term storage", and the + // second half is per-provider archive policy, not chain truth — so + // it sweeps (architecture/svm/error_normalizer.go). Only mark a + // verdict non-retryable when EVERY provider must return it by + // construction; "this node does not have it" never qualifies. + // + // Stop by breaking, not returning: the caller then sees exactly + // the result a full sweep would have produced (bestResp if one + // was collected, otherwise the aggregated ErrUpstreamsExhausted), + // just without the wasted round-trips. + if err != nil && !common.IsRetryableTowardNetwork(err) { + break + } } - loopSpan.End() - - // An error explicitly marked non-retryable toward the network is - // an authoritative answer — e.g. an OP-Stack sequencer rate limit - // every provider shares. Every remaining upstream returns the same - // verdict, so sweeping them burns latency and quota for nothing. - // Same stop condition shouldRetryWithReason already applies on the - // retry path. - // - // A skipped slot is deliberately NOT an example here. Solana's - // -32009 reads "skipped, OR missing in long-term storage", and the - // second half is per-provider archive policy, not chain truth — so - // it sweeps (architecture/svm/error_normalizer.go). Only mark a - // verdict non-retryable when EVERY provider must return it by - // construction; "this node does not have it" never qualifies. - // - // Stop by breaking, not returning: the caller then sees exactly - // the result a full sweep would have produced (bestResp if one - // was collected, otherwise the aggregated ErrUpstreamsExhausted), - // just without the wasted round-trips. - if err != nil && !common.IsRetryableTowardNetwork(err) { - break + + // Check context after the loop — handles single-upstream case where + // the loop cap is reached before a new iteration can check ctx. + if ctxErr := execSpanCtx.Err(); ctxErr != nil { + cause := context.Cause(execSpanCtx) + if cause == nil { + cause = ctxErr + } + return nil, cause } - } - // Check context after the loop — handles single-upstream case where - // the loop cap is reached before a new iteration can check ctx. - if ctxErr := execSpanCtx.Err(); ctxErr != nil { - cause := context.Cause(execSpanCtx) - if cause == nil { - cause = ctxErr + // Fallback escape: once the routed upstreams are exhausted without + // a usable result, sweep the untried fallback tier once per + // request. Errors non-retryable toward the network do not + // escalate, and consensus keeps its fixed participant set. + if (bestResp == nil || (isEmptyNetworkResult(method, bestResp) && emptyIsMiss(execSpanCtx))) && + lastErr != nil && common.IsRetryableTowardNetwork(lastErr) && + n.cfg.Failover.Enabled() && !failsafeExecutor.HasConsensus() { + // A use-upstream pin narrows the escape exactly as it narrows + // NextUpstream. + var selector string + if d := effectiveReq.Directives(); d != nil { + selector = d.UseUpstream + } + var fallbacks []common.Upstream + for _, fb := range n.upstreamsRegistry.GetFallbackEscapeUpstreams(execSpanCtx, n.networkId, method) { + if selector != "" { + if matches, _ := common.UpstreamMatchesSelector(selector, fb); !matches { + continue + } + } + if _, seen := attempted[fb.Id()]; !seen { + fallbacks = append(fallbacks, fb) + } + } + if len(fallbacks) > 0 && effectiveReq.MarkEscalatedToFallbacks() { + // Keep bestResp: if every fallback fails, the routed + // upstreams' empty answer is still the result. + lastErr = nil + maxLoopIterations = len(fallbacks) + nextUpstream = func() (common.Upstream, error) { + for len(fallbacks) > 0 { + fb := fallbacks[0] + fallbacks = fallbacks[1:] + // Skip one a sibling hedge is already running. + if _, loaded := effectiveReq.ConsumedUpstreams.LoadOrStore(fb, true); !loaded { + return fb, nil + } + } + return nil, common.NewErrNoUpstreamsLeftToSelect(effectiveReq, "no more fallback upstreams left") + } + + telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues( + n.projectId, n.Label(), method, + ).Inc() + lg.Debug().Int("fallbacks", maxLoopIterations).Msg("escalating to fallback upstreams") + + continue escalationLoop + } } - return nil, cause + + break escalationLoop } // All upstreams tried. Return the best result for the retry/hedge @@ -3120,6 +3289,12 @@ func (n *Network) handleMultiplexing(ctx context.Context, lg *zerolog.Logger, re return nil, nil, nil } + // A non-retryable write (e.g. creating a filter) must reach an upstream + // once per call, so it is never shared with a concurrent identical call. + if method, _ := req.Method(); evm.IsNonRetryableWriteMethod(method) || svm.IsNonRetryableWriteMethod(method) { + return nil, nil, nil + } + mlxHash, err := n.multiplexKey(ctx, req) lg.Trace().Str("hash", mlxHash).Object("request", req).Msgf("checking if multiplexing is possible") if err != nil || mlxHash == "" { @@ -3468,3 +3643,96 @@ func (n *Network) acquireRateLimitPermit(ctx context.Context, req *common.Normal return nil } + +// tierUpstreamsByGroup moves fallback-tier upstreams behind the rest, +// preserving order within each tier. +func tierUpstreamsByGroup(ups []common.Upstream) []common.Upstream { + return stablePartition(ups, func(u common.Upstream) bool { + return u.Config() != nil && u.Config().HasTag(common.TagTierFallback) + }) +} + +// partitionUpstreamsByLatestBlock moves upstreams whose polled head is known +// to be below bn behind the rest, preserving order within each group. An +// upstream with no known head keeps its place. +func partitionUpstreamsByLatestBlock(ups []common.Upstream, bn int64) []common.Upstream { + if bn <= 0 || len(ups) < 2 { + return ups + } + return stablePartition(ups, func(u common.Upstream) bool { + lb := upstreamLatestBlock(u) + return lb > 0 && lb < bn + }) +} + +// stablePartition returns ups with the elements matching last moved to the +// end, preserving relative order; ups itself when none match. +func stablePartition(ups []common.Upstream, last func(common.Upstream) bool) []common.Upstream { + if !slices.ContainsFunc(ups, last) { + return ups + } + out := make([]common.Upstream, 0, len(ups)) + for _, u := range ups { + if !last(u) { + out = append(out, u) + } + } + for _, u := range ups { + if last(u) { + out = append(out, u) + } + } + return out +} + +// preferTipLeaderForNearTipGetBlock moves the upstream whose head is strictly +// ahead of every other to the front for eth_getBlockByNumber of that head or +// the next block, which its siblings may not have imported yet. +func preferTipLeaderForNearTipGetBlock(ups []common.Upstream, method string, bn int64) []common.Upstream { + if method != "eth_getBlockByNumber" || len(ups) < 2 { + return ups + } + leader, tip, runnerUp := -1, int64(0), int64(0) + for i, u := range ups { + if lb := upstreamLatestBlock(u); lb > tip { + leader, tip, runnerUp = i, lb, tip + } else if lb > runnerUp { + runnerUp = lb + } + } + if leader <= 0 || tip == runnerUp || bn < tip || bn > tip+1 { + return ups + } + out := make([]common.Upstream, 0, len(ups)) + out = append(out, ups[leader]) + out = append(out, ups[:leader]...) + out = append(out, ups[leader+1:]...) + return out +} + +// upstreamLatestBlock is u's polled head, or 0 when unknown. +func upstreamLatestBlock(u common.Upstream) int64 { + if eu, ok := u.(common.EvmUpstream); ok { + if sp := eu.EvmStatePoller(); sp != nil && !sp.IsObjectNull() { + return sp.LatestBlock() + } + } + return 0 +} + +// requestBlockNumber returns the block number a request targets, or 0 when +// it has none, extracted the same way as checkUpstreamBlockAvailability. +func requestBlockNumber(ctx context.Context, req *common.NormalizedRequest) int64 { + if req == nil { + return 0 + } + if v := req.EvmBlockNumber(); v != nil { + if n64, ok := v.(int64); ok && n64 > 0 { + return n64 + } + } + if _, x, ebn := evm.ExtractBlockReferenceFromRequest(ctx, req); ebn == nil && x > 0 { + return x + } + return 0 +} diff --git a/erpc/networks_block_partition_test.go b/erpc/networks_block_partition_test.go new file mode 100644 index 000000000..b83c45c81 --- /dev/null +++ b/erpc/networks_block_partition_test.go @@ -0,0 +1,105 @@ +package erpc + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" +) + +// upWithLatest returns a FakeUpstream whose poller reports latest. +func upWithLatest(id string, latest int64) common.Upstream { + poller := common.NewFakeEvmStatePoller(latest, 0) + return common.NewFakeUpstream(id, common.WithEvmStatePoller(poller)) +} + +func ids(ups []common.Upstream) []string { + out := make([]string, len(ups)) + for i, u := range ups { + out[i] = u.Id() + } + return out +} + +func TestPartitionUpstreamsByLatestBlock_AllHaveTheBlock_NoReorder(t *testing.T) { + in := []common.Upstream{ + upWithLatest("a", 100), + upWithLatest("b", 105), + upWithLatest("c", 110), + } + got := partitionUpstreamsByLatestBlock(in, 100) + assert.Equal(t, []string{"a", "b", "c"}, ids(got), "no upstream lags the requested block; partition is identity") +} + +func TestPartitionUpstreamsByLatestBlock_NoneHaveTheBlock_NoReorder(t *testing.T) { + in := []common.Upstream{ + upWithLatest("a", 50), + upWithLatest("b", 60), + upWithLatest("c", 70), + } + got := partitionUpstreamsByLatestBlock(in, 100) + assert.Equal(t, []string{"a", "b", "c"}, ids(got), "no upstream has the requested block; partition is identity (caller can still try them)") +} + +func TestPartitionUpstreamsByLatestBlock_SplitsAndPreservesIntraGroupOrder(t *testing.T) { + in := []common.Upstream{ + upWithLatest("a", 95), + upWithLatest("b", 110), + upWithLatest("c", 90), + upWithLatest("d", 105), + upWithLatest("e", 88), + } + got := partitionUpstreamsByLatestBlock(in, 100) + assert.Equal(t, []string{"b", "d", "a", "c", "e"}, ids(got)) +} + +func TestPartitionUpstreamsByLatestBlock_EqualLatestIsTreatedAsHavingBlock(t *testing.T) { + in := []common.Upstream{ + upWithLatest("a", 99), + upWithLatest("b", 100), + upWithLatest("c", 100), + } + got := partitionUpstreamsByLatestBlock(in, 100) + assert.Equal(t, []string{"b", "c", "a"}, ids(got)) +} + +func TestPartitionUpstreamsByLatestBlock_UnknownHeadKeepsItsPlace(t *testing.T) { + in := []common.Upstream{ + common.NewFakeUpstream("a"), + upWithLatest("b", 90), + upWithLatest("c", 110), + upWithLatest("d", 0), + } + got := partitionUpstreamsByLatestBlock(in, 100) + assert.Equal(t, []string{"a", "c", "d", "b"}, ids(got), "only a known head below the block moves back") +} + +func TestPartitionUpstreamsByLatestBlock_ZeroOrNegativeBlockIsNoOp(t *testing.T) { + in := []common.Upstream{ + upWithLatest("a", 100), + upWithLatest("b", 50), + } + assert.Equal(t, in, partitionUpstreamsByLatestBlock(in, 0), "no block reference means no per-block routing preference") + assert.Equal(t, in, partitionUpstreamsByLatestBlock(in, -1), "negative block numbers are nonsensical; skip the partition rather than mis-sort") +} + +func TestPartitionUpstreamsByLatestBlock_SingleUpstreamIsNoOp(t *testing.T) { + in := []common.Upstream{upWithLatest("a", 50)} + got := partitionUpstreamsByLatestBlock(in, 100) + assert.Equal(t, in, got, "no other upstream to prefer; partition is a no-op") +} + +func TestPreferTipLeaderForNearTipGetBlock(t *testing.T) { + const method = "eth_getBlockByNumber" + + tied := []common.Upstream{upWithLatest("c", 1000), upWithLatest("b", 1000), upWithLatest("a", 1000)} + assert.Equal(t, []string{"c", "b", "a"}, ids(preferTipLeaderForNearTipGetBlock(tied, method, 1000)), + "tied heads keep the selection policy's order") + + ahead := []common.Upstream{upWithLatest("c", 999), upWithLatest("b", 999), upWithLatest("a", 1000)} + assert.Equal(t, []string{"a", "c", "b"}, ids(preferTipLeaderForNearTipGetBlock(ahead, method, 1001)), + "a strict leader goes first for its next block") + assert.Equal(t, []string{"c", "b", "a"}, ids(preferTipLeaderForNearTipGetBlock(ahead, method, 998)), + "blocks every candidate has keep the policy order") + assert.Equal(t, []string{"c", "b", "a"}, ids(preferTipLeaderForNearTipGetBlock(ahead, "eth_call", 1000))) +} diff --git a/erpc/networks_consensus_test.go b/erpc/networks_consensus_test.go index 977729eba..502afc2d2 100644 --- a/erpc/networks_consensus_test.go +++ b/erpc/networks_consensus_test.go @@ -697,6 +697,7 @@ func TestConsensusPolicy(t *testing.T) { }, retryPolicy: &common.RetryPolicyConfig{ MaxAttempts: 2, + Delay: common.Duration(50 * time.Millisecond), }, mockResponses: []mockResponse{ {status: 200, body: jsonRpcError(-32603, "internal server error")}, diff --git a/erpc/networks_earliest_detection_test.go b/erpc/networks_earliest_detection_test.go index 05360ff39..e7fdad764 100644 --- a/erpc/networks_earliest_detection_test.go +++ b/erpc/networks_earliest_detection_test.go @@ -614,11 +614,10 @@ func TestEarliestDetection_StaleHighValueInSharedState(t *testing.T) { ssr, _ := data.NewSharedStateRegistry(ctx, &log.Logger, sharedStateCfg) // Pre-compute the key that will be used for earliest block storage - // This matches what UniqueUpstreamKey computes: id + "/" + sha256(id + endpoint + networkId) + // This matches what UniqueUpstreamKey computes: id + "/" + sha256(id + endpoint) sha := sha256.New() sha.Write([]byte("rpc1")) // upCfg.Id sha.Write([]byte("http://rpc1.localhost")) // upCfg.Endpoint - sha.Write([]byte("evm:123")) // networkId uniqueKey := "rpc1/" + hex.EncodeToString(sha.Sum(nil)) presetKey := fmt.Sprintf("%s/earliestBlock/%s/blockHeader", data.CounterValueSchemaVersion, uniqueKey) t.Logf("Pre-computed key: %s", presetKey) diff --git a/erpc/networks_failover_escape_test.go b/erpc/networks_failover_escape_test.go new file mode 100644 index 000000000..5f7f35357 --- /dev/null +++ b/erpc/networks_failover_escape_test.go @@ -0,0 +1,780 @@ +package erpc + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/health" + "github.com/erpc/erpc/internal/policy" + policystdlib "github.com/erpc/erpc/internal/policy/stdlib" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/thirdparty" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + promUtil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockJsonRpcUpstream wires the state-poller mocks for one upstream at a +// fixed block height. +func mockJsonRpcUpstream(host string, chainIdHex, latestHex, finalizedHex string) { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_chainId") + }). + Reply(200). + JSON([]byte(fmt.Sprintf(`{"result":"%s"}`, chainIdHex))) + + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + b := util.SafeReadBody(r) + return strings.Contains(b, "eth_getBlockByNumber") && strings.Contains(b, `"latest"`) + }). + Reply(200). + JSON([]byte(fmt.Sprintf(`{"result":{"number":"%s","timestamp":"0x6702a8f0"}}`, latestHex))) + + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + b := util.SafeReadBody(r) + return strings.Contains(b, "eth_getBlockByNumber") && strings.Contains(b, `"finalized"`) + }). + Reply(200). + JSON([]byte(fmt.Sprintf(`{"result":{"number":"%s","timestamp":"0x6702a8e0"}}`, finalizedHex))) + + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_syncing") + }). + Reply(200). + JSON([]byte(`{"result":false}`)) +} + +// mockEthCallReturning wires an eth_call mock whose result identifies the +// upstream that served it. +func mockEthCallReturning(host string, resultHex string) { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_call") + }). + Reply(200). + JSON([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"result":"%s"}`, resultHex))) +} + +// headBoundedAvailability makes an upstream serve only blocks up to its +// polled head; the availability gate has no such implicit bound. +func headBoundedAvailability() *common.EvmBlockAvailabilityConfig { + return &common.EvmBlockAvailabilityConfig{ + Upper: &common.EvmAvailabilityBoundConfig{LatestBlockMinus: i64(0)}, + } +} + +// failoverUpstreamConfigs builds 2 primaries and 2 fallback-tier upstreams on +// rpc1..rpc4.localhost. +func failoverUpstreamConfigs() []*common.UpstreamConfig { + var cfgs []*common.UpstreamConfig + for i, id := range []string{"primary-1", "primary-2", "fallback-1", "fallback-2"} { + cfg := &common.UpstreamConfig{ + Type: common.UpstreamTypeEvm, + Id: id, + Endpoint: fmt.Sprintf("http://rpc%d.localhost", i+1), + Evm: &common.EvmUpstreamConfig{ + ChainId: 999, + StatePollerInterval: common.Duration(100 * time.Millisecond), + StatePollerDebounce: common.Duration(20 * time.Millisecond), + BlockAvailability: headBoundedAvailability(), + }, + } + if strings.HasPrefix(id, "fallback") { + cfg.Tags = []string{common.TagTierFallback} + } + cfgs = append(cfgs, cfg) + } + return cfgs +} + +// buildFailoverNetwork wires a Network with the default selection policy +// (fallback tier cordoned while a primary is healthy) on a frozen ticker; +// callers drive ticks via policy.TickForTest. +func buildFailoverNetwork( + t *testing.T, ctx context.Context, + upstreamConfigs []*common.UpstreamConfig, + opts failoverFixtureOpts, +) (*Network, *upstream.UpstreamsRegistry, *health.Tracker) { + t.Helper() + + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) + require.NoError(t, err) + // 5s metrics window so accumulated request/failure samples don't get + // reset mid-test. + mt := health.NewTracker(&log.Logger, "main", 5*time.Second) + vr := thirdparty.NewVendorsRegistry() + pr, err := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) + require.NoError(t, err) + + sharedStateCfg := &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: common.DriverMemory, + Memory: &common.MemoryConnectorConfig{ + MaxItems: 100_000, + MaxTotalSize: "1GB", + }, + }, + LockMaxWait: common.Duration(200 * time.Millisecond), + UpdateMaxWait: common.Duration(200 * time.Millisecond), + FallbackTimeout: common.Duration(3 * time.Second), + LockTtl: common.Duration(4 * time.Second), + } + require.NoError(t, sharedStateCfg.SetDefaults("test")) + ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, sharedStateCfg) + require.NoError(t, err) + + upr := upstream.NewUpstreamsRegistry( + ctx, &log.Logger, "main", upstreamConfigs, + ssr, rlr, vr, pr, nil, mt, nil, + ) + + networkConfig := &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ + ChainId: 999, + EnforceBlockAvailability: util.BoolPtr(true), + }, + // Default selection policy, frozen ticker; tests drive ticks. + SelectionPolicy: &common.SelectionPolicyConfig{ + EvalInterval: 0, + }, + } + if opts.enableFailover { + networkConfig.Failover = &common.FailoverConfig{OnDefaultsExhausted: util.BoolPtr(true)} + } + networkConfig.Failsafe = opts.failsafe + + var policyEngine *policy.Engine + if !opts.noPolicy { + policyEngine = policy.NewEngine(ctx, &log.Logger, "main", mt, policystdlib.Install, nil) + } + + network, err := NewNetwork(ctx, &log.Logger, "main", networkConfig, rlr, upr, mt, policyEngine) + require.NoError(t, err) + + upr.Bootstrap(ctx) + time.Sleep(200 * time.Millisecond) + require.NoError(t, upr.GetInitializer().WaitForTasks(ctx)) + require.NoError(t, upr.PrepareUpstreamsForNetwork(ctx, util.EvmNetworkId(999))) + require.NoError(t, network.Bootstrap(ctx)) + + upsList := upr.GetNetworkUpstreams(ctx, util.EvmNetworkId(999)) + require.Len(t, upsList, 4) + for _, ups := range upsList { + require.NoError(t, ups.Bootstrap(ctx)) + } + + // Let the pollers run a couple of cycles. + time.Sleep(500 * time.Millisecond) + + return network, upr, mt +} + +// failoverFixtureOpts configures the failoverUpstreamConfigs layout. +type failoverFixtureOpts struct { + primaryLatest string + fallbackLatest string + enableFailover bool + failsafe []*common.FailsafeConfig + // noPolicy leaves the network without a policy engine, so every + // upstream (fallbacks included) is in the routed list. + noPolicy bool + // mocks registers test-specific mocks ahead of the standard ones, before + // any poller starts. + mocks func() +} + +func setupFailoverFixture( + t *testing.T, ctx context.Context, opts failoverFixtureOpts, +) (*Network, []*upstream.Upstream, *health.Tracker) { + t.Helper() + util.ResetGock() + + const chainIdHex = "0x3e7" // 999 + const finalizedHex = "0x3e0" + + if opts.mocks != nil { + opts.mocks() + } + mockJsonRpcUpstream("rpc1.localhost", chainIdHex, opts.primaryLatest, finalizedHex) + mockJsonRpcUpstream("rpc2.localhost", chainIdHex, opts.primaryLatest, finalizedHex) + mockJsonRpcUpstream("rpc3.localhost", chainIdHex, opts.fallbackLatest, finalizedHex) + mockJsonRpcUpstream("rpc4.localhost", chainIdHex, opts.fallbackLatest, finalizedHex) + + mockEthCallReturning("rpc1.localhost", "0x1111") + mockEthCallReturning("rpc2.localhost", "0x2222") + mockEthCallReturning("rpc3.localhost", "0x3333") + mockEthCallReturning("rpc4.localhost", "0x4444") + + network, upr, mt := buildFailoverNetwork(t, ctx, failoverUpstreamConfigs(), opts) + + upsList := upr.GetNetworkUpstreams(ctx, util.EvmNetworkId(999)) + require.Len(t, upsList, 4) + + // With healthy primaries the default policy cordons the fallbacks, so + // only the per-request escape brings them in. + if network.policyEngine != nil { + policy.ResetSlotStateForTest(network.policyEngine, network.networkId, "*") + policy.TickForTest(network.policyEngine, network.networkId, "*") + } + require.NoError(t, upr.RefreshUpstreamNetworkMethodScores()) + time.Sleep(50 * time.Millisecond) + + return network, upsList, mt +} + +// ethCallRequest constructs an eth_call request targeting a specific block. +func ethCallRequest(id int, blockHex string) *common.NormalizedRequest { + return common.NewNormalizedRequest([]byte(fmt.Sprintf( + `{"jsonrpc":"2.0","id":%d,"method":"eth_call","params":[{"to":"0xdead","data":"0x"},"%s"]}`, + id, blockHex, + ))) +} + +// Once the primaries are exhausted with retryable errors, the same request +// escalates to the fallback tier. +func TestFailover_EscapeHatch(t *testing.T) { + t.Run("EscapesToFallbackOnFirstFailingRequest", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Primaries at 1000 skip block 1002; the fallbacks at 1002 serve it. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3ea", // 1002 + enableFailover: true, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + + req := ethCallRequest(1, "0x3ea") + req.SetNetwork(network) + resp, err := network.Forward(ctx, req) + require.NoError(t, err, "client request must succeed on first try via escape hatch") + require.NotNil(t, resp) + defer resp.Release() + + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + result := strings.Trim(jrr.GetResultString(), `"`) + assert.Contains(t, []string{"0x3333", "0x4444"}, result, + "escape hatch must route to a fallback; got %q", result) + + after := promUtil.ToFloat64(counter) + assert.Equal(t, before+1, after, + "MetricNetworkFallbackEscapeTotal must increment by exactly 1 for the escape firing") + }) + + t.Run("NoEscapeWhenPrimariesHealthy", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3ea", // 1002 + fallbackLatest: "0x3ea", // 1002 + enableFailover: true, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + + for i := 0; i < 30; i++ { + req := ethCallRequest(i, "0x3ea") + req.SetNetwork(network) + resp, err := network.Forward(ctx, req) + require.NoError(t, err, "healthy primary request must succeed (iter %d)", i) + require.NotNil(t, resp) + + jrr, _ := resp.JsonRpcResponse() + result := strings.Trim(jrr.GetResultString(), `"`) + assert.Contains(t, []string{"0x1111", "0x2222"}, result, + "healthy request must be served by a primary (0x1111/0x2222), got %q (iter %d)", result, i) + resp.Release() + } + + after := promUtil.ToFloat64(counter) + assert.Equal(t, before, after, + "escape hatch must NOT fire when primaries are healthy; counter must be unchanged") + }) + + t.Run("NoEscapeWhenFailoverDisabled", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // The default selection policy may still route to the fallback tier + // on its own; only the escape counter is asserted. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3ea", // 1002 + enableFailover: false, // <-- disabled + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + req := ethCallRequest(1, "0x3ea") + req.SetNetwork(network) + resp, _ := network.Forward(ctx, req) + if resp != nil { + resp.Release() + } + + after := promUtil.ToFloat64(counter) + assert.Equal(t, before, after, + "with failover disabled the per-request escape hatch must NOT fire; counter must be unchanged") + }) + + t.Run("OnlyEscalatesOncePerRequest", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Every upstream skips block 1002 (retryable); the escape fires once + // and does not re-escalate after the fallbacks also skip. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3e8", // also 1000 + enableFailover: true, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + + req := ethCallRequest(1, "0x3ea") + req.SetNetwork(network) + resp, _ := network.Forward(ctx, req) + if resp != nil { + resp.Release() + } + + after := promUtil.ToFloat64(counter) + assert.Equal(t, before+1, after, + "escape must fire EXACTLY once per request (no re-escalation loop); got %v→%v", + before, after) + }) + + t.Run("EscapesOnNonRetryableGateSkip", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Block 9000 is beyond MaxRetryableBlockDistance of the primaries at + // 1000, so their skips are non-retryable; the escape must still fire. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x2710", // 10000 + enableFailover: true, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + + req := ethCallRequest(1, "0x2328") // 9000 + req.SetNetwork(network) + resp, err := network.Forward(ctx, req) + require.NoError(t, err, + "request must succeed via fallback even when primary gate-skip is non-retryable "+ + "(distance > MaxRetryableBlockDistance)") + require.NotNil(t, resp) + defer resp.Release() + + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + result := strings.Trim(jrr.GetResultString(), `"`) + assert.Contains(t, []string{"0x3333", "0x4444"}, result, + "non-retryable-gate-skip escape must route to a fallback; got %q", result) + + // The selection policy may already exclude primaries this far behind, + // leaving the escape nothing to do. + after := promUtil.ToFloat64(counter) + assert.LessOrEqual(t, after-before, float64(1), + "escape hatch must fire at most once for the non-retryable gate-skip case") + }) + + t.Run("EscapesOnEmptyishGetBlockByNumber", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Every upstream reports 1002, but the primaries return null for it. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3ea", // 1002 + fallbackLatest: "0x3ea", // 1002 + enableFailover: true, + mocks: func() { + nullBlock := `{"jsonrpc":"2.0","id":1,"result":null}` + okBlock := `{"jsonrpc":"2.0","id":1,"result":{"number":"0x3ea","hash":"0xabc","parentHash":"0xdef","timestamp":"0x6702a8f0"}}` + for _, host := range []string{"rpc1.localhost", "rpc2.localhost"} { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + b := util.SafeReadBody(r) + return strings.Contains(b, "eth_getBlockByNumber") && strings.Contains(b, `"0x3ea"`) + }). + Reply(200). + JSON([]byte(nullBlock)) + } + for _, host := range []string{"rpc3.localhost", "rpc4.localhost"} { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + b := util.SafeReadBody(r) + return strings.Contains(b, "eth_getBlockByNumber") && strings.Contains(b, `"0x3ea"`) + }). + Reply(200). + JSON([]byte(okBlock)) + } + }, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_getBlockByNumber") + before := promUtil.ToFloat64(counter) + + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x3ea",false]}`, + )) + req.SetNetwork(network) + resp, err := network.Forward(ctx, req) + require.NoError(t, err, "null from primaries must escalate to fallbacks on the same request") + require.NotNil(t, resp) + defer resp.Release() + + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + require.False(t, jrr.IsResultEmptyish(), "fallback must return a non-null block header") + num, err := jrr.PeekStringByPath(ctx, "number") + require.NoError(t, err) + assert.Equal(t, "0x3ea", num) + + assert.Contains(t, []string{"fallback-1", "fallback-2"}, resp.UpstreamId(), + "emptyish primary miss must be served by a fallback") + + after := promUtil.ToFloat64(counter) + assert.Equal(t, before+1, after, + "escape hatch must fire for emptyish eth_getBlockByNumber primary misses") + }) + + t.Run("NoEscapeOnErrorNonRetryableTowardNetwork", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Every provider returns the same verdict for this error, so sweeping + // the fallbacks cannot help. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", + fallbackLatest: "0x3e8", + enableFailover: true, + mocks: func() { + mockEthCallError("rpc1.localhost", -32000, "sender is over rate limit") + mockEthCallError("rpc2.localhost", -32000, "sender is over rate limit") + }, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + req := ethCallRequest(1, "0x3e8") + req.SetNetwork(network) + resp, _ := network.Forward(ctx, req) + if resp != nil { + resp.Release() + } + assert.Equal(t, before, promUtil.ToFloat64(counter)) + }) + + t.Run("NoEscapeOnNullReceipt", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // A pending tx's receipt is null on every upstream; that is an + // answer, not an outage. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", + fallbackLatest: "0x3e8", + enableFailover: true, + mocks: func() { + for _, host := range []string{"rpc1.localhost", "rpc2.localhost", "rpc3.localhost", "rpc4.localhost"} { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getTransactionReceipt") + }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":null}`)) + } + }, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_getTransactionReceipt") + before := promUtil.ToFloat64(counter) + for i := 0; i < 5; i++ { + req := common.NewNormalizedRequest([]byte(fmt.Sprintf( + `{"jsonrpc":"2.0","id":%d,"method":"eth_getTransactionReceipt","params":["0x%064x"]}`, i, i+1, + ))) + req.SetNetwork(network) + resp, _ := network.Forward(ctx, req) + if resp != nil { + resp.Release() + } + } + assert.Equal(t, before, promUtil.ToFloat64(counter)) + }) + + t.Run("UseUpstreamPinLimitsTheEscape", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Primaries at 1000 skip block 1002; the fallbacks at 1002 could serve + // it, but a pin that excludes them must keep the request off them. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3ea", // 1002 + enableFailover: true, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + + req := ethCallRequest(1, "0x3ea") + req.SetNetwork(network) + req.SetDirectives(&common.RequestDirectives{UseUpstream: "primary-1"}) + resp, err := network.Forward(ctx, req) + if resp != nil { + assert.NotContains(t, []string{"fallback-1", "fallback-2"}, resp.UpstreamId(), + "a request pinned to primary-1 must not be served by a fallback") + resp.Release() + } + assert.Error(t, err, "the pinned upstream cannot serve the block, so the request fails") + assert.Equal(t, before, promUtil.ToFloat64(counter), "a pin that excludes the fallbacks must not escape") + + // A pin that still admits the fallbacks keeps the escape. + req = ethCallRequest(2, "0x3ea") + req.SetNetwork(network) + req.SetDirectives(&common.RequestDirectives{UseUpstream: "!primary-2"}) + resp, err = network.Forward(ctx, req) + require.NoError(t, err) + require.NotNil(t, resp) + defer resp.Release() + assert.Contains(t, []string{"fallback-1", "fallback-2"}, resp.UpstreamId()) + assert.Equal(t, before+1, promUtil.ToFloat64(counter)) + }) + + t.Run("KeepsTheRoutedEmptyAnswerWhenFallbacksFail", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // The primaries answer null for block 1002 and the fallbacks fail + // retryably, with no retry left: the client must still get the null. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3ea", // 1002 + fallbackLatest: "0x3ea", // 1002 + enableFailover: true, + failsafe: []*common.FailsafeConfig{{ + MatchMethod: "*", + Retry: &common.RetryPolicyConfig{MaxAttempts: 1}, + }}, + mocks: func() { + isBlock := func(r *http.Request) bool { + b := util.SafeReadBody(r) + return strings.Contains(b, "eth_getBlockByNumber") && strings.Contains(b, `"0x3ea"`) + } + for _, host := range []string{"rpc1.localhost", "rpc2.localhost"} { + gock.New("http://" + host).Post("").Persist().Filter(isBlock). + Reply(200).JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":null}`)) + } + for _, host := range []string{"rpc3.localhost", "rpc4.localhost"} { + gock.New("http://" + host).Post("").Persist().Filter(isBlock). + Reply(503).JSON([]byte(`{"jsonrpc":"2.0","id":1,"error":{"code":-32603,"message":"unavailable"}}`)) + } + }, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_getBlockByNumber") + before := promUtil.ToFloat64(counter) + + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x3ea",false]}`, + )) + req.SetNetwork(network) + resp, err := network.Forward(ctx, req) + require.NoError(t, err, "failed fallbacks must not replace the primaries' answer with an error") + require.NotNil(t, resp) + defer resp.Release() + + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.True(t, jrr.IsResultEmptyish()) + assert.Contains(t, []string{"primary-1", "primary-2"}, resp.UpstreamId()) + assert.Equal(t, before+1, promUtil.ToFloat64(counter), "the escape must still have been tried") + }) + + t.Run("DoesNotResweepFallbacksAlreadyTried", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Without a policy engine the fallbacks are already in the routed + // list, so the sweep has tried them before the escape is considered. + var fallbackHits atomic.Int64 + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", + fallbackLatest: "0x3e8", + enableFailover: true, + noPolicy: true, + mocks: func() { + for _, host := range []string{"rpc1.localhost", "rpc2.localhost", "rpc3.localhost", "rpc4.localhost"} { + host := host + fallback := host == "rpc3.localhost" || host == "rpc4.localhost" + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + // Filters run before host matching. + if r.URL.Host != host || !strings.Contains(util.SafeReadBody(r), "eth_call") { + return false + } + if fallback { + fallbackHits.Add(1) + } + return true + }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"error":{"code":-32603,"message":"internal error"}}`)) + } + }, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + req := ethCallRequest(1, "0x3e8") + req.SetNetwork(network) + resp, _ := network.Forward(ctx, req) + if resp != nil { + resp.Release() + } + assert.Equal(t, int64(2), fallbackHits.Load(), "each fallback is tried once") + assert.Equal(t, before, promUtil.ToFloat64(counter), "nothing is left to escape to") + }) + + t.Run("ConcurrentHedgesEscalateAtMostOnce", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", + fallbackLatest: "0x3e8", + enableFailover: true, + failsafe: []*common.FailsafeConfig{{ + MatchMethod: "*", + Hedge: &common.HedgePolicyConfig{Delay: common.NewStaticDuration(10 * time.Millisecond), MaxCount: 2}, + Retry: &common.RetryPolicyConfig{MaxAttempts: 2}, + }}, + mocks: func() { + for _, host := range []string{"rpc1.localhost", "rpc2.localhost"} { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_call") + }). + Reply(200). + Delay(30 * time.Millisecond). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"error":{"code":-32603,"message":"internal error"}}`)) + } + }, + }) + + counter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + before := promUtil.ToFloat64(counter) + const requests = 10 + var wg sync.WaitGroup + for i := 0; i < requests; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + req := ethCallRequest(100+i, "0x3e8") + req.SetNetwork(network) + resp, _ := network.Forward(ctx, req) + if resp != nil { + resp.Release() + } + }(i) + } + wg.Wait() + assert.LessOrEqual(t, promUtil.ToFloat64(counter)-before, float64(requests)) + }) +} + +// mockEthCallError wires an eth_call mock that returns a JSON-RPC error. +func mockEthCallError(host string, code int, message string) { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_call") + }). + Reply(200). + JSON([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"error":{"code":%d,"message":"%s"}}`, code, message))) +} + +// A consensus slot whose upstream is skipped by block availability reports +// ErrUpstreamsExhausted, which consensus counts as "no attempt", rather than +// the raw skip error. +func TestNetwork_ConsensusSlotSkippedByBlockAvailability(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3e8", + failsafe: []*common.FailsafeConfig{{ + MatchMethod: "*", + Consensus: &common.ConsensusPolicyConfig{MaxParticipants: 2, AgreementThreshold: 2}, + }}, + }) + + req := ethCallRequest(1, "0x3ea") // 1002: beyond every primary's served range + req.SetNetwork(network) + resp, err := network.Forward(ctx, req) + if resp != nil { + resp.Release() + } + assert.True(t, common.HasErrorCode(err, common.ErrCodeUpstreamsExhausted), "got %v", err) +} diff --git a/erpc/networks_finality_interpolation_test.go b/erpc/networks_finality_interpolation_test.go index 3b2f305ac..ac04135df 100644 --- a/erpc/networks_finality_interpolation_test.go +++ b/erpc/networks_finality_interpolation_test.go @@ -5,11 +5,13 @@ import ( "net/http" "strings" "testing" + "time" "github.com/erpc/erpc/architecture/evm" "github.com/erpc/erpc/common" "github.com/erpc/erpc/util" "github.com/h2non/gock" + "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -174,3 +176,66 @@ func TestInterpolation_NumericBlockFinality(t *testing.T) { finalityAfter := req.Finality(ctx) assert.NotEqual(t, common.DataFinalityStateRealtime, finalityAfter, "Numeric block should still NOT be realtime after normalization") } + +// On a chain whose finalized block equals its latest (instant finality), a +// translated "latest" must still be realtime and must not be served from a +// finalized cache policy: a nonce or balance read at "latest" has to reflect +// the chain now, not a cached answer for the block it was translated to. +func TestInterpolation_InstantFinalityLatestStaysRealtime(t *testing.T) { + for _, method := range []string{"eth_getTransactionCount", "eth_getBalance"} { + t.Run(method, func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + mockJsonRpcUpstream("rpc1.localhost", "0x7b", "0x11118888", "0x11118888") + for _, result := range []string{"0x1", "0x2"} { + gock.New("http://rpc1.localhost"). + Post(""). + Times(1). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, method) && !strings.Contains(body, `"latest"`) + }). + Reply(200). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 1, "result": result}) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _ := setupTestNetworkForInterpolation(t, ctx, nil) + require.Equal(t, network.EvmHighestLatestBlockNumber(ctx), network.EvmHighestFinalizedBlockNumber(ctx)) + + cacheCfg := &common.CacheConfig{ + Connectors: []*common.ConnectorConfig{{ + Id: "mem", + Driver: common.DriverMemory, + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }}, + Policies: []*common.CachePolicyConfig{{ + Network: "*", + Method: "*", + Finality: common.DataFinalityStateFinalized, + TTL: common.FixedDuration(time.Hour), + Connector: "mem", + }}, + } + require.NoError(t, cacheCfg.SetDefaults()) + cache, err := evm.NewEvmJsonRpcCache(ctx, &log.Logger, cacheCfg) + require.NoError(t, err) + network.cacheDal = cache.WithProjectId("prjA") + + for _, want := range []string{"0x1", "0x2"} { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"` + method + `","params":["0xabc","latest"]}`)) + req.SetNetwork(network) + resp, err := network.Forward(ctx, req) + require.NoError(t, err) + assert.Equal(t, common.DataFinalityStateRealtime, req.Finality(ctx)) + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.Equal(t, `"`+want+`"`, jrr.GetResultString()) + resp.Release() + time.Sleep(100 * time.Millisecond) // let the async cache write land + } + }) + } +} diff --git a/erpc/networks_multiplexer_test.go b/erpc/networks_multiplexer_test.go index 6cd4614bf..eb06b960b 100644 --- a/erpc/networks_multiplexer_test.go +++ b/erpc/networks_multiplexer_test.go @@ -459,3 +459,22 @@ func setupTestNetworkForMultiplexer(t *testing.T, ctx context.Context) *Network return network } + +// Non-retryable writes (EVM and SVM) never share an in-flight call; reads do. +func TestNetwork_Multiplexer_SkipsNonRetryableWrites(t *testing.T) { + n := &Network{ + cfg: &common.NetworkConfig{Architecture: common.ArchitectureEvm}, + inFlightRequests: &sync.Map{}, + } + cases := map[string]bool{ + "eth_newFilter": true, + "sendTransaction": true, + "eth_call": false, + } + for method, skip := range cases { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"` + method + `","params":[]}`)) + mlx, _, err := n.handleMultiplexing(context.Background(), &log.Logger, req, time.Now()) + require.NoError(t, err) + assert.Equal(t, skip, mlx == nil, method) + } +} diff --git a/erpc/networks_registry.go b/erpc/networks_registry.go index 1a3b93f5b..5b3f75018 100644 --- a/erpc/networks_registry.go +++ b/erpc/networks_registry.go @@ -175,6 +175,17 @@ func NewNetwork( nwCfg.Architecture = common.ArchitectureEvm } + // Keyed per project since each project has its own upstreams; tolerates + // the same rollback as upstream state pollers. + if upstreamsRegistry != nil { + if ssr := upstreamsRegistry.SharedStateRegistry(); ssr != nil { + network.latestBlockShared = ssr.GetCounterInt64( + fmt.Sprintf("network/%s/%s/latestBlock", projectId, netId), + evm.DefaultToleratedBlockHeadRollback, + ) + } + } + // Wire the architecture handler so per-network hooks dispatch correctly. // prepareNetwork also sets this; keeping it here means tests that construct // networks via NewNetwork directly get the same behavior as production. diff --git a/erpc/networks_served_tip_guaranteedfor_test.go b/erpc/networks_served_tip_guaranteedfor_test.go index 0d6869015..3314e9543 100644 --- a/erpc/networks_served_tip_guaranteedfor_test.go +++ b/erpc/networks_served_tip_guaranteedfor_test.go @@ -199,3 +199,33 @@ func TestGuaranteedFor_AppliesToTheFinalizedAxis(t *testing.T) { assert.Equal(t, int64(880), network.EvmHighestFinalizedBlockNumber(ctx), "finalized majority is 900; the internals have only finalized 880") } + +// A head already delivered to a WebSocket subscriber floors "latest", but it +// can never lift the tip past a guarantee: the named group still cannot serve +// the delivered block, and the guarantee is the operator's hard clamp. +func TestGuaranteedFor_DeliveredHeadFloorDoesNotLiftPastTheGuarantee(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}), + &common.EvmServedTipConfig{EnabledFor: []string{"latest"}, GuaranteedFor: []string{tagInternal}}) + n.NoteObservedLatestBlock(ctx, 1000) + assert.Equal(t, int64(993), n.EvmHighestLatestBlockNumber(ctx), + "the internals only have 993; a delivered 1000 must not override the guarantee") +} + +// Without a guarantee the delivered-head floor still applies on top of the +// majority served tip. +func TestGuaranteedFor_DeliveredHeadFloorAppliesWhenUnset(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + n := guaranteedForNetwork(t, ctx, mixedPool([]int64{1000, 1000, 1000}, []int64{993, 993}), + &common.EvmServedTipConfig{EnabledFor: []string{"latest"}}) + n.NoteObservedLatestBlock(ctx, 1001) + assert.Equal(t, int64(1001), n.EvmHighestLatestBlockNumber(ctx)) +} diff --git a/erpc/networks_test.go b/erpc/networks_test.go index 8f9211f8c..753ae88f7 100644 --- a/erpc/networks_test.go +++ b/erpc/networks_test.go @@ -12305,3 +12305,53 @@ func TestNetwork_CacheEmptyBehavior(t *testing.T) { cache.AssertExpectations(t) }) } + +func newStubUpstream(id, tag string) common.Upstream { + if tag == "" { + return common.NewFakeUpstream(id) + } + return common.NewFakeUpstream(id, common.WithTags(tag)) +} + +func TestTierUpstreamsByGroup(t *testing.T) { + cases := []struct { + name string + in []common.Upstream + want []string + }{ + { + name: "no fallback keeps order", + in: []common.Upstream{newStubUpstream("a", ""), newStubUpstream("b", "")}, + want: []string{"a", "b"}, + }, + { + name: "defaults before fallbacks, order kept within tier", + in: []common.Upstream{ + newStubUpstream("fallback-hi", common.TagTierFallback), + newStubUpstream("default-lo", ""), + newStubUpstream("default-hi", ""), + newStubUpstream("fallback-lo", common.TagTierFallback), + }, + want: []string{"default-lo", "default-hi", "fallback-hi", "fallback-lo"}, + }, + { + name: "only fallbacks", + in: []common.Upstream{newStubUpstream("fb-1", common.TagTierFallback), newStubUpstream("fb-2", common.TagTierFallback)}, + want: []string{"fb-1", "fb-2"}, + }, + { + name: "other tags stay in the default tier", + in: []common.Upstream{ + newStubUpstream("fb", common.TagTierFallback), + newStubUpstream("custom", "experimental"), + newStubUpstream("def", ""), + }, + want: []string{"custom", "def", "fb"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, ids(tierUpstreamsByGroup(tc.in))) + }) + } +} diff --git a/erpc/networks_tip_floor_test.go b/erpc/networks_tip_floor_test.go new file mode 100644 index 000000000..efcc94dc2 --- /dev/null +++ b/erpc/networks_tip_floor_test.go @@ -0,0 +1,72 @@ +package erpc + +import ( + "context" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" +) + +// newTipFloorTestNetwork builds the minimal Network applyDeliveredHeadFloor +// needs. With no metricsTracker the bound is defaultMaxRetryableBlockDistance. +func newTipFloorTestNetwork() *Network { + logger := zerolog.Nop() + return &Network{logger: &logger} +} + +func TestDeliveredHeadFloor_NoDeliveriesServesComputedHeadAsIs(t *testing.T) { + n := newTipFloorTestNetwork() + + assert.Equal(t, int64(100), n.applyDeliveredHeadFloor(100, 100)) + // The computed head is never remembered: a later, lower pick (fail-open, + // small reorg) passes straight through. + assert.Equal(t, int64(95), n.applyDeliveredHeadFloor(95, 95)) + assert.Equal(t, int64(0), n.applyDeliveredHeadFloor(0, 0), "an unknown head stays unknown") +} + +func TestDeliveredHeadFloor_SmallLagIsFlooredToDeliveredHead(t *testing.T) { + n := newTipFloorTestNetwork() + + // A head already delivered on a subscription sits one block ahead of + // what the pollers corroborate. + n.NoteObservedLatestBlock(context.Background(), 1001) + assert.Equal(t, int64(1001), n.applyDeliveredHeadFloor(1000, 1001), + "a computed head just below the delivered head must be floored") + assert.Equal(t, int64(1005), n.applyDeliveredHeadFloor(1005, 1005), + "once the computed head passes the floor it is served as computed") +} + +func TestDeliveredHeadFloor_BoundIsMeasuredAgainstFreshestLiveHead(t *testing.T) { + n := newTipFloorTestNetwork() + + // A stalled upstream drags the corroborated head far below the freshest + // one, which the delivered head came from. + n.NoteObservedLatestBlock(context.Background(), 9001) + assert.Equal(t, int64(9001), n.applyDeliveredHeadFloor(1000, 9000), + "a floor within reach of the freshest live head is honoured") + + // A floor no live upstream comes close to is ignored. + n.deliveredLatestBlock.Store(9000 + defaultMaxRetryableBlockDistance + 1) + assert.Equal(t, int64(1000), n.applyDeliveredHeadFloor(1000, 9000)) + + // Exactly at the bound is still honoured. + n.deliveredLatestBlock.Store(9000 + defaultMaxRetryableBlockDistance) + assert.Equal(t, int64(9000+defaultMaxRetryableBlockDistance), n.applyDeliveredHeadFloor(1000, 9000)) +} + +func TestDeliveredHeadFloor_NoLiveHeadAdoptsDeliveredHead(t *testing.T) { + n := newTipFloorTestNetwork() + + // With no known head, the delivered head is used regardless of distance. + n.NoteObservedLatestBlock(context.Background(), 5_000_000) + assert.Equal(t, int64(5_000_000), n.applyDeliveredHeadFloor(0, 0)) +} + +func TestNoteObservedLatestBlock_IsMonotonic(t *testing.T) { + n := newTipFloorTestNetwork() + n.NoteObservedLatestBlock(context.Background(), 200) + n.NoteObservedLatestBlock(context.Background(), 150) + n.NoteObservedLatestBlock(context.Background(), 0) + assert.Equal(t, int64(200), n.deliveredLatestBlock.Load()) +} From 0708176b9b6d703a50103ecd863f9144d42ceab5 Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 13:28:31 +0100 Subject: [PATCH 06/21] feat(indexer): transport-neutral event indexer for subscriptions The indexer sits between event ingresses and egresses: - ingresses push StreamEvents; newHeads from every source update that upstream's poller before dedup, then heads are delivered once and in order per network; - filter subscriptions (logs, newPendingTransactions) are refcounted per params key so clients share one upstream subscription per upstream, with subscribe and teardown serialised per filter; - logs are deduplicated by blockHash/transactionHash/logIndex, with the upstream's own removed flag tracked per log so add, remove and re-add are each delivered once; - adapters: wsupstream (WebSocket upstream ingress with resubscribe on reconnect), wsclient (per-connection egress with bounded per- subscription buffers) and nullingress (fixture ingress for tests). Co-authored-by: snowkide Co-Authored-By: Claude Opus 5.5 --- indexer/adapters/nullingress/adapter.go | 67 ++ indexer/adapters/wsclient/adapter.go | 261 +++++++ indexer/adapters/wsclient/adapter_test.go | 192 +++++ indexer/adapters/wsupstream/adapter.go | 455 +++++++++++ .../wsupstream/adapter_reconnect_test.go | 367 +++++++++ indexer/adapters/wsupstream/adapter_test.go | 90 +++ indexer/constants.go | 16 + indexer/dedup.go | 112 +++ indexer/dedup_test.go | 137 ++++ indexer/egress.go | 14 + indexer/event.go | 62 ++ indexer/filters.go | 57 ++ indexer/filters_test.go | 101 +++ indexer/indexer.go | 363 +++++++++ indexer/indexer_test.go | 738 ++++++++++++++++++ indexer/ingress.go | 42 + indexer/integration_test.go | 109 +++ 17 files changed, 3183 insertions(+) create mode 100644 indexer/adapters/nullingress/adapter.go create mode 100644 indexer/adapters/wsclient/adapter.go create mode 100644 indexer/adapters/wsclient/adapter_test.go create mode 100644 indexer/adapters/wsupstream/adapter.go create mode 100644 indexer/adapters/wsupstream/adapter_reconnect_test.go create mode 100644 indexer/adapters/wsupstream/adapter_test.go create mode 100644 indexer/constants.go create mode 100644 indexer/dedup.go create mode 100644 indexer/dedup_test.go create mode 100644 indexer/egress.go create mode 100644 indexer/event.go create mode 100644 indexer/filters.go create mode 100644 indexer/filters_test.go create mode 100644 indexer/indexer.go create mode 100644 indexer/indexer_test.go create mode 100644 indexer/ingress.go create mode 100644 indexer/integration_test.go diff --git a/indexer/adapters/nullingress/adapter.go b/indexer/adapters/nullingress/adapter.go new file mode 100644 index 000000000..475dfbe84 --- /dev/null +++ b/indexer/adapters/nullingress/adapter.go @@ -0,0 +1,67 @@ +// Package nullingress is a transport-free indexer.EventIngress fed by Push, +// used to drive the indexer in tests. +package nullingress + +import ( + "context" + "sync" + + "github.com/erpc/erpc/indexer" +) + +// Adapter implements indexer.EventIngress as an in-memory feed. +type Adapter struct { + name string + + mu sync.Mutex + started bool + sink indexer.Sink + events chan indexer.StreamEvent +} + +func New(name string) *Adapter { + return &Adapter{ + name: name, + events: make(chan indexer.StreamEvent, 64), + } +} + +func (a *Adapter) Name() string { return a.name } + +// Start forwards Push()ed events to sink until ctx is done. +func (a *Adapter) Start(ctx context.Context, _ indexer.NetworkHandle, sink indexer.Sink) error { + a.mu.Lock() + defer a.mu.Unlock() + if a.started { + return nil + } + a.sink = sink + a.started = true + go a.pump(ctx) + return nil +} + +func (a *Adapter) pump(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case ev := <-a.events: + a.sink.Ingest(ev) + } + } +} + +func (a *Adapter) EnsureFilter(context.Context, string, string, []interface{}) error { + return nil +} + +func (a *Adapter) RemoveFilter(context.Context, string, string) error { + return nil +} + +// Push injects an event. It blocks when the buffer is full, so a stalled +// indexer shows up as a test timeout rather than silent drops. +func (a *Adapter) Push(ev indexer.StreamEvent) { + a.events <- ev +} diff --git a/indexer/adapters/wsclient/adapter.go b/indexer/adapters/wsclient/adapter.go new file mode 100644 index 000000000..99b9d3353 --- /dev/null +++ b/indexer/adapters/wsclient/adapter.go @@ -0,0 +1,261 @@ +// Package wsclient adapts a client WebSocket connection into an +// indexer.EventEgress with a buffer and writer goroutine per subscription. +package wsclient + +import ( + "encoding/json" + "errors" + "sync" + + "github.com/erpc/erpc/indexer" + "github.com/rs/zerolog" +) + +// ErrClosed is returned by AddSubscription once the adapter was drained. +var ErrClosed = errors.New("connection closed") + +// ErrLimitExceeded is returned by AddSubscription when the connection +// already holds the maximum number of subscriptions. +var ErrLimitExceeded = errors.New("subscription limit exceeded") + +// NotificationWriter writes notifications to the client connection. +type NotificationWriter interface { + WriteSubscriptionNotification(clientSubId string, result json.RawMessage) error + // NotificationDropped is called each time a full buffer costs a + // notification. lossy is false when the client cannot re-derive what + // was lost (see isLossy), in which case the connection should end. + // Must not block. + NotificationDropped(sub Subscription, lossy bool) +} + +// isLossy reports whether a kind may shed its oldest queued notification +// on overflow: a newer head supersedes an older one, and pending +// transactions are best-effort by nature. Every other kind (logs, and any +// kind added later) is treated as data that must not silently vanish. +func isLossy(kind indexer.EventKind) bool { + return kind == indexer.KindNewHead || kind == indexer.KindPendingTx +} + +// Adapter is the EventEgress of one client connection. +type Adapter struct { + connID string + writer NotificationWriter + logger *zerolog.Logger + bufferSize int + + mu sync.RWMutex + drained bool + subs map[string]*clientSub // clientSubId -> sub + // routes indexes subs by event key for InterestedIn and Deliver. + routes map[routeKey]map[string]struct{} +} + +type routeKey struct { + kind indexer.EventKind + networkID string + filterHash string +} + +type clientSub struct { + id string + kind indexer.EventKind + networkID string + filterHash string // "" for newHeads + + notify chan json.RawMessage + done chan struct{} // closed by whoever removes the sub from Adapter.subs +} + +// New creates the adapter of a client connection. writer is called from +// every subscription's goroutine, so it must be safe for concurrent use. +// bufferSize is the per-subscription notification queue depth. +func New(connID string, writer NotificationWriter, logger *zerolog.Logger, bufferSize int) *Adapter { + lg := logger.With().Str("connId", connID).Logger() + return &Adapter{ + connID: connID, + writer: writer, + logger: &lg, + bufferSize: bufferSize, + subs: make(map[string]*clientSub), + routes: make(map[routeKey]map[string]struct{}), + } +} + +func (a *Adapter) Name() string { return "ws:client:" + a.connID } + +func (a *Adapter) InterestedIn(kind indexer.EventKind, networkID, filterHash string) bool { + a.mu.RLock() + defer a.mu.RUnlock() + _, ok := a.routes[routeKey{kind: kind, networkID: networkID, filterHash: filterHash}] + return ok +} + +// Deliver enqueues the event on every matching subscription without +// blocking; see enqueue for what happens when a buffer is full. +func (a *Adapter) Deliver(ev indexer.IndexedEvent) { + a.mu.RLock() + routes := a.routes[routeKey{kind: ev.Kind, networkID: ev.NetworkId, filterHash: ev.FilterHash}] + subs := make([]*clientSub, 0, len(routes)) + for id := range routes { + subs = append(subs, a.subs[id]) + } + a.mu.RUnlock() + + for _, sub := range subs { + a.deliverTo(sub, ev.Payload) + } +} + +// deliverTo enqueues payload on sub and reports a drop only while sub is +// still registered: a subscription removed after Deliver matched it has +// stopped consuming, and its queue is discarded, so nothing was lost. +func (a *Adapter) deliverTo(sub *clientSub, payload json.RawMessage) { + if enqueue(sub, payload) { + return + } + a.mu.RLock() + live := a.subs[sub.id] == sub + a.mu.RUnlock() + if live { + a.writer.NotificationDropped(sub.snapshot(), isLossy(sub.kind)) + } +} + +// AddSubscription registers a subscription and starts its writer. It fails +// with ErrLimitExceeded when the connection already holds maxSubs, and with +// ErrClosed once the adapter was drained. +func (a *Adapter) AddSubscription(clientSubID, networkID string, kind indexer.EventKind, filterHash string, maxSubs int) error { + sub := &clientSub{ + id: clientSubID, + kind: kind, + networkID: networkID, + filterHash: filterHash, + notify: make(chan json.RawMessage, a.bufferSize), + done: make(chan struct{}), + } + + a.mu.Lock() + if a.drained { + a.mu.Unlock() + return ErrClosed + } + if len(a.subs) >= maxSubs { + a.mu.Unlock() + return ErrLimitExceeded + } + a.subs[clientSubID] = sub + key := routeKey{kind: kind, networkID: networkID, filterHash: filterHash} + set, ok := a.routes[key] + if !ok { + set = make(map[string]struct{}) + a.routes[key] = set + } + set[clientSubID] = struct{}{} + a.mu.Unlock() + + go a.runWriter(sub) + return nil +} + +// RemoveSubscription removes a subscription, stops its writer, and returns +// what the caller needs to release its filter. +func (a *Adapter) RemoveSubscription(clientSubID string) (kind indexer.EventKind, networkID, filterHash string, existed bool) { + a.mu.Lock() + sub, ok := a.subs[clientSubID] + if !ok { + a.mu.Unlock() + return 0, "", "", false + } + delete(a.subs, clientSubID) + key := routeKey{kind: sub.kind, networkID: sub.networkID, filterHash: sub.filterHash} + if set, ok := a.routes[key]; ok { + delete(set, clientSubID) + if len(set) == 0 { + delete(a.routes, key) + } + } + a.mu.Unlock() + + close(sub.done) + return sub.kind, sub.networkID, sub.filterHash, true +} + +// Drain removes every subscription and makes later AddSubscription calls +// fail. It returns the subscriptions it removed, so the caller releases +// exactly those; a concurrent RemoveSubscription owns any it removed first. +func (a *Adapter) Drain() []Subscription { + a.mu.Lock() + subs := a.subs + a.drained = true + a.subs = make(map[string]*clientSub) + a.routes = make(map[routeKey]map[string]struct{}) + a.mu.Unlock() + + out := make([]Subscription, 0, len(subs)) + for _, sub := range subs { + close(sub.done) + out = append(out, sub.snapshot()) + } + return out +} + +func (sub *clientSub) snapshot() Subscription { + return Subscription{ + ClientSubID: sub.id, + Kind: sub.kind, + NetworkID: sub.networkID, + FilterHash: sub.filterHash, + } +} + +// Subscription describes one subscription. +type Subscription struct { + ClientSubID string + Kind indexer.EventKind + NetworkID string + FilterHash string +} + +// Count returns the number of active subscriptions. +func (a *Adapter) Count() int { + a.mu.RLock() + defer a.mu.RUnlock() + return len(a.subs) +} + +func (a *Adapter) runWriter(sub *clientSub) { + for { + select { + case <-sub.done: + return + case payload := <-sub.notify: + if err := a.writer.WriteSubscriptionNotification(sub.id, payload); err != nil { + a.logger.Debug().Err(err).Str("clientSubId", sub.id). + Msg("failed to write subscription notification") + } + } + } +} + +// enqueue pushes a payload onto the sub's buffer without blocking. When the +// buffer is full a lossy sub evicts its oldest notification to make room; +// any other sub drops the new one. Returns false if a notification was +// dropped either way. +func enqueue(sub *clientSub, payload json.RawMessage) bool { + dropped := false + for { + select { + case sub.notify <- payload: + return !dropped + default: + } + if !isLossy(sub.kind) { + return false + } + select { + case <-sub.notify: + dropped = true + default: + } + } +} diff --git a/indexer/adapters/wsclient/adapter_test.go b/indexer/adapters/wsclient/adapter_test.go new file mode 100644 index 000000000..bd693b7c4 --- /dev/null +++ b/indexer/adapters/wsclient/adapter_test.go @@ -0,0 +1,192 @@ +package wsclient + +import ( + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/indexer" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeWriter struct { + mu sync.Mutex + written []string + block chan struct{} // writes wait on it while non-nil + dropped []bool // lossy flag per NotificationDropped call +} + +func (w *fakeWriter) WriteSubscriptionNotification(_ string, result json.RawMessage) error { + if w.block != nil { + <-w.block + } + w.mu.Lock() + defer w.mu.Unlock() + w.written = append(w.written, string(result)) + return nil +} + +func (w *fakeWriter) NotificationDropped(_ Subscription, lossy bool) { + w.mu.Lock() + defer w.mu.Unlock() + w.dropped = append(w.dropped, lossy) +} + +func (w *fakeWriter) snapshot() (written []string, dropped []bool) { + w.mu.Lock() + defer w.mu.Unlock() + return append([]string(nil), w.written...), append([]bool(nil), w.dropped...) +} + +func newTestAdapter(w *fakeWriter, bufferSize int) *Adapter { + lg := zerolog.Nop() + return New("c1", w, &lg, bufferSize) +} + +func deliver(a *Adapter, kind indexer.EventKind, filterHash string, n int) { + for i := 0; i < n; i++ { + a.Deliver(indexer.IndexedEvent{StreamEvent: indexer.StreamEvent{ + Kind: kind, NetworkId: "evm:1", FilterHash: filterHash, Payload: json.RawMessage(fmt.Sprintf("%d", i)), + }}) + } +} + +func TestAdapter_AddAfterDrainIsRefused(t *testing.T) { + a := newTestAdapter(&fakeWriter{}, 8) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + + removed := a.Drain() + require.Len(t, removed, 1) + assert.Equal(t, "s1", removed[0].ClientSubID) + + assert.ErrorIs(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10), ErrClosed) + assert.Equal(t, 0, a.Count()) +} + +// A sub removed by RemoveSubscription is not returned by a later Drain, so +// its filter reference is released exactly once. +func TestAdapter_RemoveAndDrainAreExclusive(t *testing.T) { + a := newTestAdapter(&fakeWriter{}, 8) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + require.NoError(t, a.AddSubscription("s2", "evm:1", indexer.KindLog, "f", 10)) + + _, _, _, existed := a.RemoveSubscription("s1") + require.True(t, existed) + _, _, _, existed = a.RemoveSubscription("s1") + assert.False(t, existed) + + removed := a.Drain() + require.Len(t, removed, 1) + assert.Equal(t, "s2", removed[0].ClientSubID) +} + +func TestAdapter_SubscriptionLimitIsAtomic(t *testing.T) { + a := newTestAdapter(&fakeWriter{}, 8) + var ok atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + if a.AddSubscription(fmt.Sprintf("s%d", i), "evm:1", indexer.KindNewHead, "", 10) == nil { + ok.Add(1) + } + }(i) + } + wg.Wait() + assert.EqualValues(t, 10, ok.Load()) + assert.Equal(t, 10, a.Count()) + a.Drain() +} + +// A burst within the buffer reaches a client that keeps up. +func TestAdapter_BurstWithinBufferIsDelivered(t *testing.T) { + w := &fakeWriter{} + a := newTestAdapter(w, 256) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + defer a.Drain() + + deliver(a, indexer.KindLog, "f", 200) + require.Eventually(t, func() bool { + written, _ := w.snapshot() + return len(written) == 200 + }, 2*time.Second, 10*time.Millisecond) + _, dropped := w.snapshot() + assert.Empty(t, dropped) +} + +// Overflow on a lossy kind evicts the oldest queued notification and +// reports it as lossy. +func TestAdapter_NewHeadsOverflowDropsOldest(t *testing.T) { + w := &fakeWriter{block: make(chan struct{})} + a := newTestAdapter(w, 2) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindNewHead, "", 10)) + defer a.Drain() + + // The writer takes one event and blocks on it; two more fill the buffer. + deliver(a, indexer.KindNewHead, "", 1) + time.Sleep(50 * time.Millisecond) + deliver(a, indexer.KindNewHead, "", 5) + close(w.block) + + require.Eventually(t, func() bool { + written, _ := w.snapshot() + return len(written) == 3 + }, 2*time.Second, 10*time.Millisecond) + written, dropped := w.snapshot() + assert.Equal(t, []string{"0", "3", "4"}, written, "newest heads must survive") + assert.Equal(t, []bool{true, true, true}, dropped) +} + +// Overflow on logs keeps what is queued and reports the loss as not lossy, +// so the transport can end the connection. +func TestAdapter_LogsOverflowIsReportedNotEvicted(t *testing.T) { + w := &fakeWriter{block: make(chan struct{})} + a := newTestAdapter(w, 2) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + defer a.Drain() + + deliver(a, indexer.KindLog, "f", 1) + time.Sleep(50 * time.Millisecond) + deliver(a, indexer.KindLog, "f", 5) + close(w.block) + + require.Eventually(t, func() bool { + written, _ := w.snapshot() + return len(written) == 3 + }, 2*time.Second, 10*time.Millisecond) + written, dropped := w.snapshot() + assert.Equal(t, []string{"0", "0", "1"}, written, "queued logs must be kept in order") + assert.Equal(t, []bool{false, false, false}, dropped) +} + +// A subscription removed after Deliver matched it has stopped consuming; a +// full buffer then must not be reported as a lost logs notification, which +// would close the client's connection. +func TestAdapter_RemovedSubOverflowIsNotReported(t *testing.T) { + w := &fakeWriter{block: make(chan struct{})} + defer close(w.block) + a := newTestAdapter(w, 2) + require.NoError(t, a.AddSubscription("s1", "evm:1", indexer.KindLog, "f", 10)) + defer a.Drain() + + deliver(a, indexer.KindLog, "f", 1) + time.Sleep(50 * time.Millisecond) // the writer holds the first entry + deliver(a, indexer.KindLog, "f", 2) + _, dropped := w.snapshot() + require.Empty(t, dropped, "the buffer is full but nothing was dropped yet") + + a.mu.RLock() + sub := a.subs["s1"] + a.mu.RUnlock() + a.RemoveSubscription("s1") + a.deliverTo(sub, json.RawMessage("x")) + + _, dropped = w.snapshot() + assert.Empty(t, dropped, "a removed subscription must not report a drop") +} diff --git a/indexer/adapters/wsupstream/adapter.go b/indexer/adapters/wsupstream/adapter.go new file mode 100644 index 000000000..d0814ea9b --- /dev/null +++ b/indexer/adapters/wsupstream/adapter.go @@ -0,0 +1,455 @@ +// Package wsupstream adapts a WebSocket JSON-RPC upstream into an +// indexer.EventIngress: it owns the upstream eth_subscribe/eth_unsubscribe +// calls and resubscribes after every reconnect. +package wsupstream + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/erpc/erpc/clients" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/indexer" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog" +) + +const ( + methodEthSubscribe = "eth_subscribe" + methodEthUnsubscribe = "eth_unsubscribe" + + // resubAttemptTimeout bounds each subscribe attempt and unsubscribe. + resubAttemptTimeout = 15 * time.Second +) + +// Resubscribe backoff bounds; vars so tests can shorten them. +var ( + resubRetryMin = 1 * time.Second + resubRetryMax = 30 * time.Second +) + +var errNotConnected = errors.New("upstream websocket is not connected") + +// Adapter is the EventIngress of one WS upstream on one network. +type Adapter struct { + upstreamID string + networkID string + wsClient *clients.WsJsonRpcClient + logger *zerolog.Logger + + // forward sends subscription RPCs through the upstream's Forward (rate + // limits, timeouts, metrics); replaced in tests. + forward func(ctx context.Context, nq *common.NormalizedRequest, bypassMethodExclusion bool) (*common.NormalizedResponse, error) + + retryMin time.Duration + retryMax time.Duration + + stripSubscribeFromBlockZero bool + + nw indexer.NetworkHandle + sink indexer.Sink + + // subsMu guards all mutable state below. Never held across an RPC. + subsMu sync.Mutex + // resubCancel cancels the resubscribe loop of connection resubEpoch. + resubCancel context.CancelFunc + resubEpoch uint64 + heads upstreamSub + // filters (by filterKey) survive disconnects to be resubscribed. + filters map[string]*filterSub +} + +// upstreamSub is one upstream subscription: newHeads or a filter. +type upstreamSub struct { + // mu serialises subscribe attempts so EnsureFilter and the resubscribe + // loop can't create duplicates. + mu sync.Mutex + // id and epoch identify the live upstream subscription (id "" when + // none); removed is set once it is dropped. Guarded by Adapter.subsMu. + id string + epoch uint64 + removed bool +} + +// Options are the network-level settings of an Adapter. +type Options struct { + // StripSubscribeFromBlockZero removes a zero fromBlock from logs + // filters; see common.EvmNetworkConfig.StripSubscribeFromBlockZero. + StripSubscribeFromBlockZero bool +} + +type filterSub struct { + upstreamSub + subType string + paramsHash string + params []interface{} +} + +// New returns the adapter of up, or nil if up is not a WebSocket upstream. +func New(up *upstream.Upstream, networkID string, logger *zerolog.Logger, opts Options) *Adapter { + wsClient, ok := up.Client.(*clients.WsJsonRpcClient) + if !ok { + return nil + } + lg := logger.With().Str("upstreamId", up.Id()).Str("networkId", networkID).Logger() + return &Adapter{ + upstreamID: up.Id(), + networkID: networkID, + wsClient: wsClient, + logger: &lg, + filters: make(map[string]*filterSub), + forward: func(ctx context.Context, nq *common.NormalizedRequest, bypassMethodExclusion bool) (*common.NormalizedResponse, error) { + return up.Forward(ctx, nq, bypassMethodExclusion, false) + }, + retryMin: resubRetryMin, + retryMax: resubRetryMax, + stripSubscribeFromBlockZero: opts.StripSubscribeFromBlockZero, + } +} + +func (a *Adapter) Name() string { return "ws:" + a.upstreamID } + +// Start registers the connection hooks and subscribes newHeads in the +// background, detached from ctx. +func (a *Adapter) Start(_ context.Context, nw indexer.NetworkHandle, sink indexer.Sink) error { + a.nw = nw + a.sink = sink + + cbID := a.Name() + a.wsClient.SetOnReconnect(cbID, func() { + a.logger.Info().Msg("upstream websocket reconnected, resubscribing") + a.startResubscribe() + }) + a.wsClient.SetOnDisconnect(cbID, func() { + a.logger.Info().Msg("upstream websocket disconnected, will resubscribe on reconnect") + // The subscriptions died with the connection; forget them so the + // next connection resubscribes everything. + a.subsMu.Lock() + if a.resubCancel != nil { + a.resubCancel() + } + a.heads.id = "" + for _, sub := range a.filters { + sub.id = "" + } + a.subsMu.Unlock() + }) + + a.startResubscribe() + return nil +} + +// startResubscribe launches the retry loop for the current connection +// unless one already runs for it. +func (a *Adapter) startResubscribe() { + epoch := a.wsClient.Epoch() + a.subsMu.Lock() + defer a.subsMu.Unlock() + if epoch == 0 || epoch == a.resubEpoch { + return + } + if a.resubCancel != nil { + a.resubCancel() + } + ctx, cancel := context.WithCancel(context.Background()) + a.resubCancel, a.resubEpoch = cancel, epoch + go a.resubscribeWithRetry(ctx) +} + +// EnsureFilter subscribes a filter; a no-op if it is already subscribed. A +// filter this call added is dropped again on failure, so it isn't +// resubscribed on every reconnect. +func (a *Adapter) EnsureFilter(ctx context.Context, subType, paramsHash string, params []interface{}) error { + key := filterKey(subType, paramsHash) + + a.subsMu.Lock() + sub, exists := a.filters[key] + if !exists { + sub = &filterSub{subType: subType, paramsHash: paramsHash, params: params} + a.filters[key] = sub + } + a.subsMu.Unlock() + + sub.mu.Lock() + defer sub.mu.Unlock() + err := a.subscribeFilterLocked(ctx, sub) + if err != nil && !exists { + a.subsMu.Lock() + if a.filters[key] == sub { + delete(a.filters, key) + } + a.subsMu.Unlock() + a.drop(ctx, &sub.upstreamSub) + } + return err +} + +// RemoveFilter unsubscribes a filter and stops resubscribing it. +func (a *Adapter) RemoveFilter(ctx context.Context, subType, paramsHash string) error { + key := filterKey(subType, paramsHash) + + a.subsMu.Lock() + sub, ok := a.filters[key] + delete(a.filters, key) + a.subsMu.Unlock() + if ok { + a.drop(ctx, &sub.upstreamSub) + } + return nil +} + +func filterKey(subType, paramsHash string) string { + return subType + ":" + paramsHash +} + +// resubscribeWithRetry (re)subscribes newHeads and every filter, retrying +// with backoff until all succeed or ctx is cancelled, so one failed +// subscribe doesn't leave the adapter without heads until the next +// reconnect. +func (a *Adapter) resubscribeWithRetry(ctx context.Context) { + backoff := a.retryMin + for { + if ctx.Err() != nil { + return + } + done := true + attemptCtx, cancel := context.WithTimeout(ctx, resubAttemptTimeout) + a.heads.mu.Lock() + err := a.subscribeLocked(attemptCtx, &a.heads, []interface{}{indexer.SubTypeNewHeads}, a.handleNewHeads) + a.heads.mu.Unlock() + cancel() + if err != nil { + done = false + a.logger.Warn().Err(err).Msg("failed to subscribe newHeads, will retry") + } + + a.subsMu.Lock() + filters := make([]*filterSub, 0, len(a.filters)) + for _, sub := range a.filters { + filters = append(filters, sub) + } + a.subsMu.Unlock() + for _, sub := range filters { + attemptCtx, cancel := context.WithTimeout(ctx, resubAttemptTimeout) + sub.mu.Lock() + err := a.subscribeFilterLocked(attemptCtx, sub) + sub.mu.Unlock() + cancel() + if err != nil { + done = false + a.logger.Warn().Err(err).Str("subType", sub.subType).Str("paramsHash", sub.paramsHash). + Msg("failed to re-subscribe filter, will retry") + } + } + if done { + a.logger.Info().Msg("all upstream subscriptions (re)established") + return + } + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + backoff = min(backoff*2, a.retryMax) + } +} + +func (a *Adapter) subscribeFilterLocked(ctx context.Context, sub *filterSub) error { + outParams := append([]interface{}{sub.subType}, sub.params[1:]...) + if a.stripSubscribeFromBlockZero { + if cleaned, changed := stripFromBlockZero(outParams); changed { + a.logger.Debug(). + Str("subType", sub.subType). + Str("paramsHash", sub.paramsHash). + Msg("stripped zero fromBlock from eth_subscribe filter") + outParams = cleaned + } + } + err := a.subscribeLocked(ctx, &sub.upstreamSub, outParams, func(params []byte) { + a.handleFilter(sub.subType, sub.paramsHash, params) + }) + if err != nil { + return fmt.Errorf("filter subscribe: %w", err) + } + return nil +} + +// subscribeLocked subscribes sub unless it is live or dropped; the caller +// holds sub.mu. The result is kept only if sub is still wanted and its +// connection is still up, and released upstream otherwise. +func (a *Adapter) subscribeLocked(ctx context.Context, sub *upstreamSub, params []interface{}, handler func(params []byte)) error { + a.subsMu.Lock() + skip := sub.id != "" || sub.removed + a.subsMu.Unlock() + if skip { + return nil + } + if !a.wsClient.IsConnected() { + return errNotConnected + } + + ws := &clients.WsSubscription{Handler: handler} + if err := a.send(clients.WithWsSubscription(ctx, ws), methodEthSubscribe, params); err != nil { + return err + } + if ws.ID == "" { + return errors.New("upstream returned no subscription id") + } + + a.subsMu.Lock() + connUp := ws.Epoch == a.wsClient.Epoch() + commit := connUp && !sub.removed + if commit { + sub.id, sub.epoch = ws.ID, ws.Epoch + } + a.subsMu.Unlock() + + if !connUp { + return errNotConnected + } + if !commit { + a.release(context.WithoutCancel(ctx), ws.ID, ws.Epoch) + return nil + } + a.logger.Info().Str("upstreamSubId", ws.ID).Interface("subType", params[0]).Msg("subscribed upstream") + return nil +} + +// drop marks sub removed, so an in-flight subscribe releases its result +// instead of committing it, and releases its live subscription if any. +func (a *Adapter) drop(ctx context.Context, sub *upstreamSub) { + a.subsMu.Lock() + id, epoch := sub.id, sub.epoch + sub.id, sub.removed = "", true + a.subsMu.Unlock() + if id != "" { + a.release(ctx, id, epoch) + } +} + +// release drops a subscription's handler and cancels it upstream. Both are +// no-ops once its connection is gone, since the subscription died with it. +func (a *Adapter) release(ctx context.Context, id string, epoch uint64) { + a.wsClient.UnregisterSubscriptionHandler(id, epoch) + ctx, cancel := context.WithTimeout(ctx, resubAttemptTimeout) + defer cancel() + _ = a.send(clients.WithWsSubscription(ctx, &clients.WsSubscription{Epoch: epoch}), methodEthUnsubscribe, []interface{}{id}) +} + +// send forwards a subscription RPC through the upstream. It is marked +// internal so upstream-scope retry and hedge policies never send it twice: +// a duplicate eth_subscribe would leave an orphan subscription. +func (a *Adapter) send(ctx context.Context, method string, params []interface{}) error { + body, err := common.SonicCfg.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "id": util.RandomID(), + "method": method, + "params": params, + }) + if err != nil { + return err + } + nq := common.NewNormalizedRequest(body) + nq.SetDirectives(&common.RequestDirectives{IsInternal: true}) + _, err = a.forward(ctx, nq, false) + return err +} + +type notificationParams struct { + Result json.RawMessage `json:"result"` +} + +func (a *Adapter) handleNewHeads(raw []byte) { + var outer notificationParams + if err := common.SonicCfg.Unmarshal(raw, &outer); err != nil { + a.logger.Warn().Err(err).Msg("failed to parse newHeads notification envelope") + return + } + var header struct { + Number string `json:"number"` + Hash string `json:"hash"` + } + if err := common.SonicCfg.Unmarshal(outer.Result, &header); err != nil { + a.logger.Warn().Err(err).Msg("failed to parse newHeads result") + return + } + num, err := common.HexToInt64(header.Number) + if err != nil { + a.logger.Warn().Err(err).Str("number", header.Number).Msg("failed to parse block number") + return + } + a.sink.Ingest(indexer.StreamEvent{ + Kind: indexer.KindNewHead, + NetworkId: a.networkID, + SourceId: a.Name(), + Block: indexer.BlockRef{Number: num, Hash: header.Hash}, + Payload: outer.Result, + }) +} + +func (a *Adapter) handleFilter(subType, paramsHash string, raw []byte) { + var outer notificationParams + if err := common.SonicCfg.Unmarshal(raw, &outer); err != nil { + a.logger.Warn().Err(err).Str("subType", subType).Msg("failed to parse filter notification envelope") + return + } + kind := indexer.KindLog + if subType == indexer.SubTypeNewPendingTransactions { + kind = indexer.KindPendingTx + } + a.sink.Ingest(indexer.StreamEvent{ + Kind: kind, + NetworkId: a.networkID, + SourceId: a.Name(), + FilterHash: paramsHash, + Payload: outer.Result, + }) +} + +// stripFromBlockZero returns a copy of params without fromBlock in filter +// objects whose fromBlock is zero. params is not mutated, so it stays valid +// for the paramsHash and later resubscribes. +func stripFromBlockZero(params []interface{}) ([]interface{}, bool) { + out := make([]interface{}, len(params)) + changed := false + for i, p := range params { + f, ok := p.(map[string]interface{}) + if !ok { + out[i] = p + continue + } + fb, hasFrom := f["fromBlock"] + if !hasFrom { + out[i] = f + continue + } + s, isStr := fb.(string) + if !isStr || !isZeroBlockRef(s) { + out[i] = f + continue + } + clean := make(map[string]interface{}, len(f)) + for k, v := range f { + if k == "fromBlock" { + continue + } + clean[k] = v + } + out[i] = clean + changed = true + } + return out, changed +} + +// isZeroBlockRef reports whether s is a zero integer literal ("0", "0x0", ...). +func isZeroBlockRef(s string) bool { + n, err := strconv.ParseInt(strings.TrimSpace(s), 0, 64) + return err == nil && n == 0 +} diff --git a/indexer/adapters/wsupstream/adapter_reconnect_test.go b/indexer/adapters/wsupstream/adapter_reconnect_test.go new file mode 100644 index 000000000..672e152c0 --- /dev/null +++ b/indexer/adapters/wsupstream/adapter_reconnect_test.go @@ -0,0 +1,367 @@ +package wsupstream + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/clients" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/indexer" + "github.com/gorilla/websocket" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --- test doubles ------------------------------------------------------- + +type fakeNetworkHandle struct{} + +func (fakeNetworkHandle) Id() string { return "evm:123" } +func (fakeNetworkHandle) SuggestLatestBlock(string, int64) {} + +type fakeSink struct { + events chan indexer.StreamEvent +} + +func (s *fakeSink) Ingest(ev indexer.StreamEvent) { s.events <- ev } + +// notifyServer is a WS upstream that answers eth_subscribe with ids from a +// per-connection counter ("0x1", "0x2", …, as real nodes do), follows each +// logs subscribe response with one notification for it, answers any other +// method with true, and can push notification frames. +type notifyServer struct { + srv *httptest.Server + newConn chan *notifyConn + // gate, when set, is received from after each eth_subscribe is + // recorded and before it is answered. + gate chan struct{} +} + +type notifyConn struct { + conn *websocket.Conn + writeMu sync.Mutex + requests chan notifyRequest +} + +type notifyRequest struct { + Method string `json:"method"` + Params []interface{} `json:"params"` + // SubID is the id this connection assigned (eth_subscribe only). + SubID string `json:"-"` +} + +func newNotifyServer(t *testing.T) *notifyServer { + n := ¬ifyServer{newConn: make(chan *notifyConn, 16)} + upgrader := websocket.Upgrader{} + n.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + nc := ¬ifyConn{conn: conn, requests: make(chan notifyRequest, 64)} + n.newConn <- nc + go nc.serve(n.gate) + })) + t.Cleanup(n.srv.Close) + return n +} + +func (n *notifyServer) wsURL(t *testing.T) *url.URL { + u, err := url.Parse(n.srv.URL) + require.NoError(t, err) + u.Scheme = "ws" + return u +} + +func (nc *notifyConn) serve(gate chan struct{}) { + subs := 0 + for { + _, msg, err := nc.conn.ReadMessage() + if err != nil { + return + } + var req struct { + notifyRequest + ID json.RawMessage `json:"id"` + } + if common.SonicCfg.Unmarshal(msg, &req) != nil { + continue + } + if req.Method != methodEthSubscribe { + nc.write(`{"jsonrpc":"2.0","id":` + string(req.ID) + `,"result":true}`) + nc.requests <- req.notifyRequest + continue + } + subs++ + req.SubID = fmt.Sprintf("0x%x", subs) + nc.requests <- req.notifyRequest + if gate != nil { + <-gate + } + nc.write(`{"jsonrpc":"2.0","id":` + string(req.ID) + `,"result":"` + req.SubID + `"}`) + if req.Params[0] == indexer.SubTypeLogs { + nc.sendLog(req.SubID, 1) + } + } +} + +func (nc *notifyConn) write(frame string) { + nc.writeMu.Lock() + defer nc.writeMu.Unlock() + _ = nc.conn.SetWriteDeadline(time.Now().Add(time.Second)) + _ = nc.conn.WriteMessage(websocket.TextMessage, []byte(frame)) +} + +func (nc *notifyConn) sendNewHead(subID string, num int64) { + nc.write(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_subscription","params":{"subscription":%q,"result":{"number":"0x%x","hash":"0xhash%x","parentHash":"0xhash%x"}}}`, + subID, num, num, num-1)) +} + +func (nc *notifyConn) sendLog(subID string, num int64) { + nc.write(fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_subscription","params":{"subscription":%q,"result":{"blockNumber":"0x%x","blockHash":"0xhash%x"}}}`, + subID, num, num)) +} + +// nextRequest returns the next request with method, skipping others. +func (nc *notifyConn) nextRequest(t *testing.T, method string) notifyRequest { + t.Helper() + timeout := time.After(3 * time.Second) + for { + select { + case req := <-nc.requests: + if req.Method == method { + return req + } + case <-timeout: + t.Fatalf("server never received %s", method) + } + } +} + +func compressResubRetry(t *testing.T) { + origMin, origMax := resubRetryMin, resubRetryMax + resubRetryMin = 20 * time.Millisecond + resubRetryMax = 100 * time.Millisecond + t.Cleanup(func() { resubRetryMin, resubRetryMax = origMin, origMax }) +} + +// newTestAdapter wires an adapter to a real WS client for server. Its +// forward sends straight through the client (the real one rides +// upstream.Forward). +func newTestAdapter(t *testing.T, u *url.URL) (*Adapter, *fakeSink) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + logger := zerolog.New(zerolog.NewTestWriter(t)).Level(zerolog.WarnLevel) + ci, err := clients.NewWsJsonRpcClient(ctx, &logger, "test-project", common.NewFakeUpstream("test-ws-upstream"), u, nil, nil) + require.NoError(t, err) + wsc := ci.(*clients.WsJsonRpcClient) + a := &Adapter{ + upstreamID: "test-ws-upstream", + networkID: "evm:123", + wsClient: wsc, + logger: &logger, + filters: make(map[string]*filterSub), + forward: func(ctx context.Context, nq *common.NormalizedRequest, _ bool) (*common.NormalizedResponse, error) { + return wsc.SendRequest(ctx, nq) + }, + retryMin: resubRetryMin, + retryMax: resubRetryMax, + } + return a, &fakeSink{events: make(chan indexer.StreamEvent, 64)} +} + +// subscribedHeads reports whether a is connected with a live newHeads +// subscription. +func subscribedHeads(a *Adapter) func() bool { + return func() bool { + if !a.wsClient.IsConnected() { + return false + } + a.subsMu.Lock() + defer a.subsMu.Unlock() + return a.heads.id != "" + } +} + +func logsParams(address string) []interface{} { + return []interface{}{indexer.SubTypeLogs, map[string]interface{}{"address": address}} +} + +func nextEvent(t *testing.T, sink *fakeSink, kind indexer.EventKind) indexer.StreamEvent { + t.Helper() + timeout := time.After(3 * time.Second) + for { + select { + case ev := <-sink.events: + if ev.Kind == kind { + return ev + } + case <-timeout: + t.Fatalf("no %v event delivered", kind) + } + } +} + +// --- tests ---------------------------------------------------------------- + +// TestAdapterResubscribesWithRetryAfterReconnect: a failed resubscribe must +// be retried rather than leave the adapter without heads until the next +// reconnect. +func TestAdapterResubscribesWithRetryAfterReconnect(t *testing.T) { + compressResubRetry(t) + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + conn1 := <-server.newConn + + var forwardCalls, failuresLeft atomic.Int64 + const failuresPerEpoch = 2 + failuresLeft.Store(failuresPerEpoch) + send := a.forward + a.forward = func(ctx context.Context, nq *common.NormalizedRequest, bypass bool) (*common.NormalizedResponse, error) { + forwardCalls.Add(1) + if failuresLeft.Add(-1) >= 0 { + return nil, errors.New("circuit breaker is open on upstream-level") + } + return send(ctx, nq, bypass) + } + + require.NoError(t, a.Start(context.Background(), fakeNetworkHandle{}, sink)) + + require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond, + "adapter never subscribed newHeads despite the failures stopping after %d", failuresPerEpoch) + require.GreaterOrEqual(t, forwardCalls.Load(), int64(failuresPerEpoch+1), + "expected the subscribe to be retried through failures") + + conn1.sendNewHead(conn1.nextRequest(t, methodEthSubscribe).SubID, 100) + require.Equal(t, int64(100), nextEvent(t, sink, indexer.KindNewHead).Block.Number) + + // Ungraceful upstream death: kill the TCP connection with no close + // handshake. The adapter must clear its stale subscription, then retry + // the resubscribe through a fresh round of failures. + failuresLeft.Store(failuresPerEpoch) + _ = conn1.conn.UnderlyingConn().Close() + var conn2 *notifyConn + select { + case conn2 = <-server.newConn: + case <-time.After(3 * time.Second): + t.Fatal("client never re-dialed after the upstream connection was killed") + } + + require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond, + "adapter never re-established the newHeads subscription after reconnect") + conn2.sendNewHead(conn2.nextRequest(t, methodEthSubscribe).SubID, 101) + require.Equal(t, int64(101), nextEvent(t, sink, indexer.KindNewHead).Block.Number) +} + +// TestAdapterResubscribesAllFiltersAfterReconnect: subscription ids come +// from a per-connection counter, so the new connection reuses the old +// ids for different subscriptions. Every filter must deliver after the +// resubscribe, and each must be subscribed exactly once. +func TestAdapterResubscribesAllFiltersAfterReconnect(t *testing.T) { + compressResubRetry(t) + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + conn1 := <-server.newConn + require.NoError(t, a.Start(context.Background(), fakeNetworkHandle{}, sink)) + require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond) + + const numFilters = 8 + for i := 0; i < numFilters; i++ { + params := logsParams(fmt.Sprintf("0x%x", i)) + require.NoError(t, a.EnsureFilter(context.Background(), indexer.SubTypeLogs, indexer.BuildParamsKey(params), params)) + } + + _ = conn1.conn.UnderlyingConn().Close() + conn2 := <-server.newConn + subscribes := make(map[string]string) // sub id -> first param + for i := 0; i < numFilters+1; i++ { + req := conn2.nextRequest(t, methodEthSubscribe) + subscribes[req.SubID] = fmt.Sprint(req.Params[0]) + } + require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond) + + for id, subType := range subscribes { + if subType == indexer.SubTypeLogs { + conn2.sendLog(id, 2) + } + } + delivered := make(map[string]bool) + for len(delivered) < numFilters { + delivered[nextEvent(t, sink, indexer.KindLog).FilterHash] = true + } + + select { + case req := <-conn2.requests: + if req.Method == methodEthSubscribe { + t.Fatalf("unexpected extra subscribe after resubscribe: %v", req.Params) + } + case <-time.After(200 * time.Millisecond): + } +} + +// TestAdapterDeliversFirstNotificationAfterSubscribe: the upstream sends a +// log right behind the subscribe response; it must not be dropped. +func TestAdapterDeliversFirstNotificationAfterSubscribe(t *testing.T) { + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + + params := logsParams("0xabc") + require.NoError(t, a.EnsureFilter(context.Background(), indexer.SubTypeLogs, indexer.BuildParamsKey(params), params)) + assert.Equal(t, indexer.BuildParamsKey(params), nextEvent(t, sink, indexer.KindLog).FilterHash) +} + +// TestAdapterRemoveFilterDuringSubscribe: RemoveFilter while the subscribe +// is in flight must not leave the late subscription behind upstream. +func TestAdapterRemoveFilterDuringSubscribe(t *testing.T) { + server := newNotifyServer(t) + server.gate = make(chan struct{}) + a, sink := newTestAdapter(t, server.wsURL(t)) + conn := <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + + params := logsParams("0xabc") + hash := indexer.BuildParamsKey(params) + ensured := make(chan error, 1) + go func() { ensured <- a.EnsureFilter(context.Background(), indexer.SubTypeLogs, hash, params) }() + subID := conn.nextRequest(t, methodEthSubscribe).SubID + + require.NoError(t, a.RemoveFilter(context.Background(), indexer.SubTypeLogs, hash)) + server.gate <- struct{}{} + require.NoError(t, <-ensured) + + unsub := conn.nextRequest(t, methodEthUnsubscribe) + assert.Equal(t, []interface{}{subID}, unsub.Params) + a.subsMu.Lock() + assert.Empty(t, a.filters) + a.subsMu.Unlock() +} + +// TestAdapterEnsureFilterWhileDisconnected: EnsureFilter must report the +// failure (so the indexer tries other ingresses) and not keep the filter. +func TestAdapterEnsureFilterWhileDisconnected(t *testing.T) { + server := newNotifyServer(t) + u := server.wsURL(t) + server.srv.Close() + a, _ := newTestAdapter(t, u) + + params := logsParams("0xabc") + err := a.EnsureFilter(context.Background(), indexer.SubTypeLogs, indexer.BuildParamsKey(params), params) + require.ErrorIs(t, err, errNotConnected) + a.subsMu.Lock() + assert.Empty(t, a.filters) + a.subsMu.Unlock() +} diff --git a/indexer/adapters/wsupstream/adapter_test.go b/indexer/adapters/wsupstream/adapter_test.go new file mode 100644 index 000000000..7c92b006f --- /dev/null +++ b/indexer/adapters/wsupstream/adapter_test.go @@ -0,0 +1,90 @@ +package wsupstream + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsZeroBlockRef(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {"0", true}, + {"0x0", true}, + {"0X0", true}, + {"0x00", true}, + {"0x0000", true}, + {" 0x0 ", true}, + {"0x1", false}, + {"0x100", false}, + {"latest", false}, + {"finalized", false}, + {"", false}, + {"0x", false}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + assert.Equal(t, c.want, isZeroBlockRef(c.in)) + }) + } +} + +func TestStripFromBlockZero(t *testing.T) { + t.Run("strips fromBlock:0x0 and keeps toBlock", func(t *testing.T) { + params := []interface{}{ + map[string]interface{}{ + "address": "0xabc", + "topics": []string{"0xdef"}, + "fromBlock": "0x0", + "toBlock": "latest", + }, + } + out, changed := stripFromBlockZero(params) + assert.True(t, changed) + m := out[0].(map[string]interface{}) + _, hasFrom := m["fromBlock"] + assert.False(t, hasFrom) + assert.Equal(t, "latest", m["toBlock"]) + assert.Equal(t, "0xabc", m["address"]) + }) + + t.Run("non-zero fromBlock passes through untouched", func(t *testing.T) { + params := []interface{}{ + map[string]interface{}{"fromBlock": "0x100", "toBlock": "latest"}, + } + out, changed := stripFromBlockZero(params) + assert.False(t, changed) + m := out[0].(map[string]interface{}) + assert.Equal(t, "0x100", m["fromBlock"]) + }) + + t.Run("no fromBlock returns changed=false", func(t *testing.T) { + params := []interface{}{ + map[string]interface{}{"address": "0xabc", "topics": []string{"0xdef"}}, + } + out, changed := stripFromBlockZero(params) + assert.False(t, changed) + assert.Equal(t, params, out) + }) + + t.Run("non-map params are untouched", func(t *testing.T) { + params := []interface{}{"logs", "notAMap", 42} + out, changed := stripFromBlockZero(params) + assert.False(t, changed) + assert.Equal(t, params, out) + }) + + t.Run("input is not mutated", func(t *testing.T) { + original := []interface{}{ + map[string]interface{}{"fromBlock": "0x0", "address": "0xabc"}, + } + _, changed := stripFromBlockZero(original) + assert.True(t, changed) + // Original still has fromBlock. + orig := original[0].(map[string]interface{}) + _, hasFrom := orig["fromBlock"] + assert.True(t, hasFrom, "stripFromBlockZero must not mutate its input") + }) +} diff --git a/indexer/constants.go b/indexer/constants.go new file mode 100644 index 000000000..ccefa8676 --- /dev/null +++ b/indexer/constants.go @@ -0,0 +1,16 @@ +// Package indexer is the transport-neutral event-stream core: dedup, +// per-filter subscription bookkeeping, and fan-out to egresses. It must not +// import erpc, clients, or any transport library, so ingress and egress +// implementations stay swappable. +package indexer + +// Supported eth_subscribe subscription types. +const ( + SubTypeNewHeads = "newHeads" + SubTypeLogs = "logs" + SubTypeNewPendingTransactions = "newPendingTransactions" +) + +// DefaultDedupWindowSize bounds the per-filter set of recently delivered +// notification keys. +const DefaultDedupWindowSize = 8192 diff --git a/indexer/dedup.go b/indexer/dedup.go new file mode 100644 index 000000000..5ef9c5a2e --- /dev/null +++ b/indexer/dedup.go @@ -0,0 +1,112 @@ +package indexer + +import ( + "encoding/json" + "strings" + "sync" + + "github.com/erpc/erpc/common" +) + +// DedupKeyForFilter returns the identity of a filter notification, or "" if +// it cannot be established (the caller then delivers without deduping). +// +// - logs: blockHash + txHash + logIndex, all required. The removed flag is +// not part of the key but the state DedupWindow tracks per key, so +// add, remove and re-add are each delivered once. +// - newPendingTransactions: the tx hash, from a string or an object's hash. +// +// Keys are lowercased because hex on the wire is case-insensitive. +func DedupKeyForFilter(subType string, result json.RawMessage) string { + switch subType { + case SubTypeLogs: + var log struct { + BlockHash string `json:"blockHash"` + TxHash string `json:"transactionHash"` + LogIndex string `json:"logIndex"` + } + if err := common.SonicCfg.Unmarshal(result, &log); err != nil { + return "" + } + if log.BlockHash == "" || log.TxHash == "" || log.LogIndex == "" { + return "" + } + return strings.ToLower(log.BlockHash + ":" + log.TxHash + ":" + log.LogIndex) + case SubTypeNewPendingTransactions: + var asString string + if err := common.SonicCfg.Unmarshal(result, &asString); err == nil && asString != "" { + return strings.ToLower(asString) + } + var asObj struct { + Hash string `json:"hash"` + } + if err := common.SonicCfg.Unmarshal(result, &asObj); err == nil { + return strings.ToLower(asObj.Hash) + } + } + return "" +} + +// logRemoved returns a log payload's "removed" flag, false if unparseable. +func logRemoved(payload json.RawMessage) bool { + if len(payload) == 0 { + return false + } + var probe struct { + Removed bool `json:"removed"` + } + if err := common.SonicCfg.Unmarshal(payload, &probe); err != nil { + return false + } + return probe.Removed +} + +// DedupWindow is a bounded FIFO map from key to its last delivered removed +// state. Keys past capacity evict the oldest. Safe for concurrent use. +type DedupWindow struct { + size int + + mu sync.Mutex + state map[string]bool + order []string +} + +// NewDedupWindow returns a DedupWindow holding up to size keys, or +// DefaultDedupWindowSize if size is not positive. +func NewDedupWindow(size int) *DedupWindow { + if size <= 0 { + size = DefaultDedupWindowSize + } + return &DedupWindow{ + size: size, + state: make(map[string]bool), + } +} + +// Mark records key in the given removed state and reports whether it should +// be delivered: the key is new or its last delivered state differs. +func (w *DedupWindow) Mark(key string, removed bool) bool { + w.mu.Lock() + defer w.mu.Unlock() + if last, ok := w.state[key]; ok { + if last == removed { + return false + } + w.state[key] = removed + return true + } + // Decoded strings may alias the notification buffer; the window + // outlives it. + key = strings.Clone(key) + w.state[key] = removed + w.order = append(w.order, key) + + if len(w.order) > w.size { + evict := len(w.order) - w.size + for _, old := range w.order[:evict] { + delete(w.state, old) + } + w.order = w.order[evict:] + } + return true +} diff --git a/indexer/dedup_test.go b/indexer/dedup_test.go new file mode 100644 index 000000000..ee30420d5 --- /dev/null +++ b/indexer/dedup_test.go @@ -0,0 +1,137 @@ +package indexer + +import ( + "encoding/json" + "testing" +) + +func TestDedupWindow_MarkNewKey(t *testing.T) { + w := NewDedupWindow(4) + if !w.Mark("a", false) { + t.Fatalf("first Mark of a new key must return true") + } + if w.Mark("a", false) { + t.Fatalf("second Mark of the same key and state must return false") + } +} + +func TestDedupWindow_StateChangeIsDelivered(t *testing.T) { + w := NewDedupWindow(4) + steps := []struct { + removed bool + want bool + }{ + {false, true}, // add + {false, false}, // duplicate add + {true, true}, // remove + {true, false}, // duplicate remove + {false, true}, // re-add + } + for i, s := range steps { + if got := w.Mark("k", s.removed); got != s.want { + t.Fatalf("step %d (removed=%v): got %v want %v", i, s.removed, got, s.want) + } + } +} + +func TestDedupWindow_EvictsOldestPastCapacity(t *testing.T) { + w := NewDedupWindow(3) + for _, k := range []string{"a", "b", "c"} { + if !w.Mark(k, false) { + t.Fatalf("Mark(%q) must be true while window has room", k) + } + } + // Window: a, b, c. Adding "d" evicts "a"; window becomes b, c, d. + if !w.Mark("d", false) { + t.Fatalf("Mark(d) must be true") + } + // "a" was evicted and is therefore newly markable. + if !w.Mark("a", false) { + t.Fatalf("a should be evicted and therefore newly markable") + } + // "c" has not been evicted yet — still a dupe. + if w.Mark("c", false) { + t.Fatalf("c should still be in-window") + } +} + +func TestDedupWindow_ZeroSizeUsesDefault(t *testing.T) { + w := NewDedupWindow(0) + if w.size != DefaultDedupWindowSize { + t.Fatalf("size with 0 arg should default to %d, got %d", DefaultDedupWindowSize, w.size) + } +} + +// An idle window must not pin capacity-sized storage: every active filter +// holds one. +func TestDedupWindow_EmptyWindowDoesNotPreallocate(t *testing.T) { + w := NewDedupWindow(DefaultDedupWindowSize) + if cap(w.order) != 0 { + t.Fatalf("empty window must not preallocate order, cap=%d", cap(w.order)) + } +} + +func TestDedupKeyForFilter_Logs(t *testing.T) { + payload := json.RawMessage(`{"blockHash":"0xabc","transactionHash":"0xdef","logIndex":"0x1","removed":false}`) + got := DedupKeyForFilter(SubTypeLogs, payload) + want := "0xabc:0xdef:0x1" + if got != want { + t.Fatalf("log dedup key: got %q want %q", got, want) + } +} + +func TestDedupKeyForFilter_LogsRemovedFlagIsNotPartOfKey(t *testing.T) { + // removed is the log's state, tracked by DedupWindow per key; the key + // is the log's identity. + in := DedupKeyForFilter(SubTypeLogs, json.RawMessage(`{"blockHash":"0xabc","transactionHash":"0xdef","logIndex":"0x1","removed":false}`)) + out := DedupKeyForFilter(SubTypeLogs, json.RawMessage(`{"blockHash":"0xabc","transactionHash":"0xdef","logIndex":"0x1","removed":true}`)) + if in != out { + t.Fatalf("removed=true / removed=false must share a key, got %q vs %q", in, out) + } +} + +func TestDedupKeyForFilter_LogsMissingIdentityReturnsEmpty(t *testing.T) { + for _, p := range []string{ + `{}`, + `{"transactionHash":"0xdef","logIndex":"0x1"}`, + `{"blockHash":"0xabc","logIndex":"0x1"}`, + `{"blockHash":"0xabc","transactionHash":"0xdef"}`, + `not json`, + } { + if got := DedupKeyForFilter(SubTypeLogs, json.RawMessage(p)); got != "" { + t.Fatalf("payload %s: want empty key, got %q", p, got) + } + } +} + +func TestDedupKeyForFilter_CaseInsensitive(t *testing.T) { + a := DedupKeyForFilter(SubTypeLogs, json.RawMessage(`{"blockHash":"0xABC","transactionHash":"0xDEF","logIndex":"0xA"}`)) + b := DedupKeyForFilter(SubTypeLogs, json.RawMessage(`{"blockHash":"0xabc","transactionHash":"0xdef","logIndex":"0xa"}`)) + if a != b { + t.Fatalf("log keys must ignore hex case, got %q vs %q", a, b) + } + if DedupKeyForFilter(SubTypeNewPendingTransactions, json.RawMessage(`"0xBEEF"`)) != "0xbeef" { + t.Fatalf("pending tx key must ignore hex case") + } +} + +func TestDedupKeyForFilter_PendingTxAsString(t *testing.T) { + got := DedupKeyForFilter(SubTypeNewPendingTransactions, json.RawMessage(`"0xbeef"`)) + if got != "0xbeef" { + t.Fatalf("pendingTx string key: got %q", got) + } +} + +func TestDedupKeyForFilter_PendingTxAsObject(t *testing.T) { + got := DedupKeyForFilter(SubTypeNewPendingTransactions, json.RawMessage(`{"hash":"0xbeef"}`)) + if got != "0xbeef" { + t.Fatalf("pendingTx object key: got %q", got) + } +} + +func TestDedupKeyForFilter_UnknownSubTypeReturnsEmpty(t *testing.T) { + got := DedupKeyForFilter("unknown", json.RawMessage(`{}`)) + if got != "" { + t.Fatalf("unknown subType should return empty key, got %q", got) + } +} diff --git a/indexer/egress.go b/indexer/egress.go new file mode 100644 index 000000000..84c8c810b --- /dev/null +++ b/indexer/egress.go @@ -0,0 +1,14 @@ +package indexer + +// EventEgress receives the IndexedEvents that survive dedup. Each egress +// owns its buffering and drop policy; the indexer delivers synchronously. +type EventEgress interface { + // Name identifies the egress; it must be unique among attached egresses. + Name() string + // InterestedIn reports whether the egress routes events with this key. + // Called for every event, so it must be cheap and must not block. + InterestedIn(kind EventKind, networkId, filterHash string) bool + // Deliver hands over an event. It must not block: a slow egress must + // only hurt itself, never other egresses or the ingress. + Deliver(ev IndexedEvent) +} diff --git a/indexer/event.go b/indexer/event.go new file mode 100644 index 000000000..1ebaa8d37 --- /dev/null +++ b/indexer/event.go @@ -0,0 +1,62 @@ +package indexer + +import "encoding/json" + +// EventKind discriminates the events flowing through the indexer. +type EventKind uint8 + +const ( + KindUnknown EventKind = iota + KindNewHead + KindLog + KindPendingTx +) + +// String returns the matching eth_subscribe subscription type. +func (k EventKind) String() string { + switch k { + case KindNewHead: + return "newHeads" + case KindLog: + return "logs" + case KindPendingTx: + return "newPendingTransactions" + default: + return "unknown" + } +} + +// BlockRef identifies a block. It is zero for events without one. +type BlockRef struct { + Number int64 + Hash string +} + +func (b BlockRef) Zero() bool { + return b.Number == 0 && b.Hash == "" +} + +// StreamEvent is what an ingress emits, before dedup: sources covering the +// same network may emit the same event. +type StreamEvent struct { + Kind EventKind + NetworkId string + // SourceId is the Name() of the ingress that produced the event. + SourceId string + Block BlockRef + // FilterHash is the BuildParamsKey of the filter the event belongs to; + // empty for KindNewHead. + FilterHash string + // Payload is the upstream's notification result, verbatim. + Payload json.RawMessage +} + +// IndexedEvent is a deduplicated StreamEvent as delivered to egresses. The +// indexer never synthesizes events: reorged-out logs reach clients only as +// the upstream's own removed:true notifications. +type IndexedEvent struct { + StreamEvent + + // Removed is the log payload's "removed" flag; false for other kinds. + Removed bool +} diff --git a/indexer/filters.go b/indexer/filters.go new file mode 100644 index 000000000..9cefbc0f5 --- /dev/null +++ b/indexer/filters.go @@ -0,0 +1,57 @@ +package indexer + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "hash/fnv" + + "github.com/erpc/erpc/common" +) + +// BuildParamsKey returns a short hash of eth_subscribe params that is stable +// across sources and processes. encoding/json is used because it sorts map +// keys. +func BuildParamsKey(params []interface{}) string { + data, err := json.Marshal(params) + if err != nil { + return fmt.Sprintf("%v", params) + } + h := fnv.New64a() + _, _ = h.Write(data) + return fmt.Sprintf("%x", h.Sum64()) +} + +// ExtractSubscriptionType returns the subscription type of eth_subscribe +// params, or "" if the first param is missing or not a string. +func ExtractSubscriptionType(params []interface{}) string { + if len(params) == 0 { + return "" + } + st, _ := params[0].(string) + return st +} + +// ExtractClientSubID returns the subscription ID of eth_unsubscribe params, +// or a subscription-not-found error if it is missing or not a string. +func ExtractClientSubID(params []interface{}) (string, error) { + if len(params) == 0 { + return "", common.NewErrSubscriptionNotFound("") + } + id, ok := params[0].(string) + if !ok { + return "", common.NewErrSubscriptionNotFound(fmt.Sprintf("%v", params[0])) + } + return id, nil +} + +// GenerateClientSubID returns a random subscription ID in the "0x" + 32 hex +// format Ethereum clients use. +func GenerateClientSubID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return "0x" + hex.EncodeToString(b), nil +} diff --git a/indexer/filters_test.go b/indexer/filters_test.go new file mode 100644 index 000000000..4a99bd805 --- /dev/null +++ b/indexer/filters_test.go @@ -0,0 +1,101 @@ +package indexer + +import ( + "strings" + "testing" +) + +func TestBuildParamsKey_StableAcrossCalls(t *testing.T) { + a := BuildParamsKey([]interface{}{"logs", map[string]interface{}{"topics": []string{"0x1"}}}) + b := BuildParamsKey([]interface{}{"logs", map[string]interface{}{"topics": []string{"0x1"}}}) + if a != b { + t.Fatalf("same params must hash identically, got %q vs %q", a, b) + } +} + +func TestBuildParamsKey_IndependentOfMapOrder(t *testing.T) { + params := []interface{}{"logs", map[string]interface{}{ + "address": "0xabc", + "topics": []interface{}{"0x1", nil}, + "fromBlock": "latest", + "toBlock": "latest", + }} + want := BuildParamsKey(params) + for i := 0; i < 200; i++ { + if got := BuildParamsKey(params); got != want { + t.Fatalf("call %d: same params hashed to %q, want %q", i, got, want) + } + } +} + +func TestBuildParamsKey_DifferentParamsDiffer(t *testing.T) { + a := BuildParamsKey([]interface{}{"logs", map[string]interface{}{"topics": []string{"0x1"}}}) + b := BuildParamsKey([]interface{}{"logs", map[string]interface{}{"topics": []string{"0x2"}}}) + if a == b { + t.Fatalf("different params must produce different keys") + } +} + +func TestExtractSubscriptionType(t *testing.T) { + cases := []struct { + name string + params []interface{} + want string + }{ + {"newHeads", []interface{}{"newHeads"}, "newHeads"}, + {"logs with filter", []interface{}{"logs", map[string]interface{}{}}, "logs"}, + {"empty params", []interface{}{}, ""}, + {"non-string first", []interface{}{123}, ""}, + } + for _, c := range cases { + if got := ExtractSubscriptionType(c.params); got != c.want { + t.Errorf("%s: got %q want %q", c.name, got, c.want) + } + } +} + +func TestExtractClientSubID_ValidString(t *testing.T) { + got, err := ExtractClientSubID([]interface{}{"0xabc"}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if got != "0xabc" { + t.Fatalf("got %q want 0xabc", got) + } +} + +func TestExtractClientSubID_EmptyParams(t *testing.T) { + _, err := ExtractClientSubID(nil) + if err == nil { + t.Fatalf("expected error on empty params") + } +} + +func TestExtractClientSubID_NonString(t *testing.T) { + _, err := ExtractClientSubID([]interface{}{123}) + if err == nil { + t.Fatalf("expected error on non-string param") + } +} + +func TestGenerateClientSubID_Shape(t *testing.T) { + id, err := GenerateClientSubID() + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if !strings.HasPrefix(id, "0x") { + t.Fatalf("id must start with 0x: %q", id) + } + // 2 for "0x" + 32 hex chars (16 bytes * 2) + if len(id) != 34 { + t.Fatalf("id must be 34 chars, got %d (%q)", len(id), id) + } +} + +func TestGenerateClientSubID_Unique(t *testing.T) { + a, _ := GenerateClientSubID() + b, _ := GenerateClientSubID() + if a == b { + t.Fatalf("two consecutive IDs must differ") + } +} diff --git a/indexer/indexer.go b/indexer/indexer.go new file mode 100644 index 000000000..e382a5d88 --- /dev/null +++ b/indexer/indexer.go @@ -0,0 +1,363 @@ +package indexer + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "github.com/rs/zerolog" +) + +// Options configures an Indexer. Zero values select the defaults. +type Options struct { + // DedupWindowSize is the per-filter dedup capacity. + DedupWindowSize int +} + +// Indexer dedupes the StreamEvents ingresses push at it and fans them out +// to every interested egress. Ingest runs synchronously on the ingress's +// goroutine, which keeps per-source ordering without extra queues. +type Indexer struct { + logger *zerolog.Logger + opts Options + + networks sync.Map // networkId -> *networkState + egresses sync.Map // egress Name() -> EventEgress +} + +// headMarker is the most recently delivered head of a network. +type headMarker struct { + num int64 + hash string +} + +// networkState holds a network's NetworkHandle, the ingresses feeding it, +// the newHeads dedup marker, and the dedup windows for each active filter. +type networkState struct { + handle NetworkHandle + + ingressMu sync.RWMutex + ingresses map[string]EventIngress // Name() -> ingress + selector IngressSelector // nil: every ingress is a default + + // headMu serialises newHeads dedup and fan-out, so concurrent sources + // can neither deliver the same head twice nor deliver heads out of + // order. Egress Deliver is non-blocking by contract. + headMu sync.Mutex + lastHead *headMarker // guarded by headMu; nil until the first head + + filterMu sync.RWMutex + filters map[string]*filterState // paramsHash -> state +} + +// filterState is one filter subscription shared by every client with the +// same params. +type filterState struct { + // mu is held across this filter's ingress calls so a subscribe and the + // last client's teardown never interleave. + mu sync.Mutex + subscribed bool // guarded by mu + refs int // guarded by networkState.filterMu + dedup *DedupWindow +} + +// New returns an empty Indexer. Events for networks not registered with +// RegisterNetwork are ignored. +func New(logger *zerolog.Logger, opts Options) *Indexer { + return &Indexer{ + logger: logger, + opts: opts, + } +} + +// RegisterNetwork registers a network; later calls for the same Id() keep +// the first handle. +func (i *Indexer) RegisterNetwork(nw NetworkHandle) { + if _, ok := i.networks.Load(nw.Id()); ok { + return + } + i.networks.LoadOrStore(nw.Id(), &networkState{ + handle: nw, + ingresses: make(map[string]EventIngress), + filters: make(map[string]*filterState), + }) +} + +func errNetworkNotRegistered(networkId string) error { + return fmt.Errorf("indexer: network %q not registered", networkId) +} + +// AddIngress adds an ingress to a registered network and starts it with +// the indexer as its Sink. +func (i *Indexer) AddIngress(ctx context.Context, networkId string, ing EventIngress) error { + nsRaw, ok := i.networks.Load(networkId) + if !ok { + return errNetworkNotRegistered(networkId) + } + ns := nsRaw.(*networkState) + ns.ingressMu.Lock() + ns.ingresses[ing.Name()] = ing + ns.ingressMu.Unlock() + return ing.Start(ctx, ns.handle, i) +} + +// Attach registers an egress until the returned detach is called. +func (i *Indexer) Attach(eg EventEgress) (detach func()) { + i.egresses.Store(eg.Name(), eg) + return func() { i.egresses.Delete(eg.Name()) } +} + +// RegisterNetworkSelector sets the IngressSelector EnsureFilter uses for a +// registered network; nil selects every ingress. +func (i *Indexer) RegisterNetworkSelector(networkId string, sel IngressSelector) { + nsRaw, ok := i.networks.Load(networkId) + if !ok { + return + } + ns := nsRaw.(*networkState) + ns.ingressMu.Lock() + ns.selector = sel + ns.ingressMu.Unlock() +} + +// EnsureFilter takes a reference on a filter, subscribing it on the +// network's ingresses if needed. Concurrent callers for the same filter wait +// for the subscribe in flight and retry it themselves if it failed. +func (i *Indexer) EnsureFilter(ctx context.Context, networkId, subType string, params []interface{}) (paramsHash string, err error) { + nsRaw, ok := i.networks.Load(networkId) + if !ok { + return "", errNetworkNotRegistered(networkId) + } + ns := nsRaw.(*networkState) + paramsHash = BuildParamsKey(params) + + ns.filterMu.Lock() + f := ns.filters[paramsHash] + if f == nil { + f = &filterState{dedup: NewDedupWindow(i.opts.DedupWindowSize)} + ns.filters[paramsHash] = f + } + f.refs++ + ns.filterMu.Unlock() + + f.mu.Lock() + defer f.mu.Unlock() + if f.subscribed { + return paramsHash, nil + } + if err := i.subscribe(ctx, ns, subType, paramsHash, params); err != nil { + // Ingresses may keep a filter whose subscribe failed (to retry on + // reconnect), so remove it everywhere. + i.removeFromIngresses(ctx, ns, subType, paramsHash) + ns.filterMu.Lock() + f.refs-- + if f.refs == 0 { + delete(ns.filters, paramsHash) + } + ns.filterMu.Unlock() + return paramsHash, err + } + f.subscribed = true + return paramsHash, nil +} + +// subscribe tries the selector's defaults, then its fallbacks only if every +// default failed. It fails when no ingress subscribed, including when none +// was selected. +func (i *Indexer) subscribe(ctx context.Context, ns *networkState, subType, paramsHash string, params []interface{}) error { + networkId := ns.handle.Id() + + ns.ingressMu.RLock() + ings := make(map[string]EventIngress, len(ns.ingresses)) + for name, ing := range ns.ingresses { + ings[name] = ing + } + sel := ns.selector + ns.ingressMu.RUnlock() + + defaults, fallbacks := partitionIngresses(sel, ings, networkId, subType, params) + + var errs []error + for _, tier := range [][]EventIngress{defaults, fallbacks} { + subscribed := false + for _, ing := range tier { + if err := ing.EnsureFilter(ctx, subType, paramsHash, params); err != nil { + errs = append(errs, err) + i.logger.Warn().Err(err).Str("ingress", ing.Name()).Str("networkId", networkId). + Str("subType", subType).Str("paramsHash", paramsHash). + Msg("ingress EnsureFilter failed") + continue + } + subscribed = true + } + if subscribed { + return nil + } + } + if len(errs) == 0 { + return fmt.Errorf("indexer: no ingress selected for %s filter on network %q", subType, networkId) + } + return errors.Join(errs...) +} + +// partitionIngresses resolves the selector's tiers to registered ingresses, +// dropping unknown and repeated names. Without a selector every ingress is +// a default. +func partitionIngresses(sel IngressSelector, ings map[string]EventIngress, networkId, subType string, params []interface{}) (defaults, fallbacks []EventIngress) { + if sel == nil { + defaults = make([]EventIngress, 0, len(ings)) + for _, ing := range ings { + defaults = append(defaults, ing) + } + return defaults, nil + } + dNames, fNames := sel.Select(networkId, subType, params) + named := make(map[string]struct{}, len(dNames)+len(fNames)) + pick := func(names []string) []EventIngress { + out := make([]EventIngress, 0, len(names)) + for _, n := range names { + if _, dup := named[n]; dup { + continue + } + named[n] = struct{}{} + if ing, ok := ings[n]; ok { + out = append(out, ing) + } + } + return out + } + return pick(dNames), pick(fNames) +} + +// ReleaseFilter drops a reference taken by EnsureFilter and unsubscribes the +// filter from every ingress once none remain. +func (i *Indexer) ReleaseFilter(ctx context.Context, networkId, subType, paramsHash string) { + nsRaw, ok := i.networks.Load(networkId) + if !ok { + return + } + ns := nsRaw.(*networkState) + + ns.filterMu.Lock() + f := ns.filters[paramsHash] + if f == nil || f.refs <= 0 { + ns.filterMu.Unlock() + return + } + f.refs-- + last := f.refs == 0 + ns.filterMu.Unlock() + if !last { + return + } + + f.mu.Lock() + defer f.mu.Unlock() + ns.filterMu.Lock() + // While waiting for f.mu a new client may have joined, or an earlier + // release already tore the filter down. + keep := f.refs > 0 || !f.subscribed + ns.filterMu.Unlock() + if keep { + return + } + i.removeFromIngresses(ctx, ns, subType, paramsHash) + f.subscribed = false + ns.filterMu.Lock() + if f.refs == 0 { + delete(ns.filters, paramsHash) + } + ns.filterMu.Unlock() +} + +// removeFromIngresses calls RemoveFilter on every ingress; those that never +// subscribed the filter no-op. +func (i *Indexer) removeFromIngresses(ctx context.Context, ns *networkState, subType, paramsHash string) { + ns.ingressMu.RLock() + ings := make([]EventIngress, 0, len(ns.ingresses)) + for _, ing := range ns.ingresses { + ings = append(ings, ing) + } + ns.ingressMu.RUnlock() + + for _, ing := range ings { + if err := ing.RemoveFilter(ctx, subType, paramsHash); err != nil { + i.logger.Warn().Err(err).Str("ingress", ing.Name()).Str("networkId", ns.handle.Id()). + Str("subType", subType).Str("paramsHash", paramsHash). + Msg("ingress RemoveFilter failed") + } + } +} + +// Ingest reports heads to the NetworkHandle, dedupes, and fans out to every +// interested egress. +func (i *Indexer) Ingest(ev StreamEvent) { + nsRaw, ok := i.networks.Load(ev.NetworkId) + if !ok { + return + } + ns := nsRaw.(*networkState) + + // Before dedup, so every source's head counts even if another source + // already delivered it. + if ev.Kind == KindNewHead && !ev.Block.Zero() && ev.SourceId != "" { + ns.handle.SuggestLatestBlock(ev.SourceId, ev.Block.Number) + } + + // The upstream's removed flag is trusted as-is. + removed := ev.Kind == KindLog && logRemoved(ev.Payload) + + if ev.Kind == KindNewHead { + ns.headMu.Lock() + defer ns.headMu.Unlock() + } + if !i.dedupe(ns, &ev, removed) { + return + } + + i.fanOut(IndexedEvent{StreamEvent: ev, Removed: removed}) +} + +// dedupe reports whether ev should be delivered. Heads are compared to the +// last delivered head; filter events go through their filter's DedupWindow. +func (i *Indexer) dedupe(ns *networkState, ev *StreamEvent, removed bool) bool { + switch ev.Kind { + case KindNewHead: + // Caller holds headMu. A different hash at the same height is a + // reorg and is delivered. + if prev := ns.lastHead; prev != nil { + if ev.Block.Number < prev.num || (ev.Block.Number == prev.num && strings.EqualFold(prev.hash, ev.Block.Hash)) { + return false + } + } + ns.lastHead = &headMarker{num: ev.Block.Number, hash: strings.Clone(ev.Block.Hash)} + return true + case KindLog, KindPendingTx: + ns.filterMu.RLock() + f := ns.filters[ev.FilterHash] + ns.filterMu.RUnlock() + if f == nil { + return true + } + key := DedupKeyForFilter(ev.Kind.String(), ev.Payload) + if key == "" { + return true + } + return f.dedup.Mark(key, removed) + default: + return true + } +} + +func (i *Indexer) fanOut(ev IndexedEvent) { + i.egresses.Range(func(_, v any) bool { + eg := v.(EventEgress) + if !eg.InterestedIn(ev.Kind, ev.NetworkId, ev.FilterHash) { + return true + } + eg.Deliver(ev) + return true + }) +} diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go new file mode 100644 index 000000000..fcbe50949 --- /dev/null +++ b/indexer/indexer_test.go @@ -0,0 +1,738 @@ +package indexer + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/rs/zerolog" +) + +// --- fakes ----------------------------------------------------------- + +type fakeNetwork struct { + id string + + mu sync.Mutex + suggestedBy map[string][]int64 // sourceId -> block nums seen +} + +func newFakeNetwork(id string) *fakeNetwork { + return &fakeNetwork{ + id: id, + suggestedBy: make(map[string][]int64), + } +} + +func (n *fakeNetwork) Id() string { return n.id } +func (n *fakeNetwork) SuggestLatestBlock(sourceId string, block int64) { + n.mu.Lock() + n.suggestedBy[sourceId] = append(n.suggestedBy[sourceId], block) + n.mu.Unlock() +} + +type fakeEgress struct { + name string + filters map[string]struct{} // filterHash -> interested + acceptAllHeads bool + // hook, when set, runs at the start of every Deliver. + hook func(IndexedEvent) + + mu sync.Mutex + received []IndexedEvent +} + +func (e *fakeEgress) Name() string { return e.name } +func (e *fakeEgress) InterestedIn(kind EventKind, networkId, filterHash string) bool { + if kind == KindNewHead { + return e.acceptAllHeads + } + _, ok := e.filters[filterHash] + return ok +} +func (e *fakeEgress) Deliver(ev IndexedEvent) { + if e.hook != nil { + e.hook(ev) + } + e.mu.Lock() + e.received = append(e.received, ev) + e.mu.Unlock() +} +func (e *fakeEgress) count() int { + e.mu.Lock() + defer e.mu.Unlock() + return len(e.received) +} + +type fakeIngress struct { + name string + ensureCalls atomic.Int32 + removeCalls atomic.Int32 + lastParamsHash atomic.Value // string + // ensureErr, when set, is returned from every EnsureFilter call. + errMu sync.Mutex + ensureErr error + // active reports whether the ingress holds the filter. Like the WS + // adapter, a failed EnsureFilter still stores it. + active atomic.Bool + // hook, when set, runs at the start of every "ensure"/"remove" call. + hook func(op string) + + startedFor NetworkHandle + sink Sink +} + +func (i *fakeIngress) Name() string { return i.name } +func (i *fakeIngress) Start(_ context.Context, nw NetworkHandle, sink Sink) error { + i.startedFor = nw + i.sink = sink + return nil +} +func (i *fakeIngress) setErr(err error) { + i.errMu.Lock() + i.ensureErr = err + i.errMu.Unlock() +} +func (i *fakeIngress) getErr() error { + i.errMu.Lock() + defer i.errMu.Unlock() + return i.ensureErr +} +func (i *fakeIngress) EnsureFilter(_ context.Context, _ string, paramsHash string, _ []interface{}) error { + if i.hook != nil { + i.hook("ensure") + } + i.ensureCalls.Add(1) + i.lastParamsHash.Store(paramsHash) + i.active.Store(true) + return i.getErr() +} +func (i *fakeIngress) RemoveFilter(_ context.Context, _, _ string) error { + if i.hook != nil { + i.hook("remove") + } + i.removeCalls.Add(1) + i.active.Store(false) + return nil +} + +// fakeSelector is a static IngressSelector for tests. +type fakeSelector struct { + defaults []string + fallbacks []string +} + +func (s *fakeSelector) Select(_, _ string, _ []interface{}) ([]string, []string) { + return s.defaults, s.fallbacks +} + +// --- tests ----------------------------------------------------------- + +func newIndexer(t *testing.T) *Indexer { + t.Helper() + logger := zerolog.New(zerolog.NewTestWriter(t)) + return New(&logger, Options{}) +} + +func TestIndexer_NewHead_FanOutAndDedup(t *testing.T) { + idx := newIndexer(t) + nw := newFakeNetwork("evm:1") + idx.RegisterNetwork(nw) + eg := &fakeEgress{name: "eg1", filters: map[string]struct{}{}, acceptAllHeads: true} + idx.Attach(eg) + + ev := StreamEvent{ + Kind: KindNewHead, + NetworkId: "evm:1", + SourceId: "ws:up1", + Block: BlockRef{Number: 100, Hash: "0xAAA"}, + } + idx.Ingest(ev) + idx.Ingest(ev) // dup + + if got := eg.count(); got != 1 { + t.Fatalf("want 1 delivery after dup, got %d", got) + } + // Second unique head advances. + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:up1", Block: BlockRef{Number: 101, Hash: "0xBBB"}}) + if got := eg.count(); got != 2 { + t.Fatalf("want 2 after advance, got %d", got) + } +} + +// Several sources delivering the same head at the same instant must reach +// the egress exactly once. +func TestIndexer_NewHead_ConcurrentIngestDedupe(t *testing.T) { + idx := newIndexer(t) + nw := newFakeNetwork("evm:1") + idx.RegisterNetwork(nw) + eg := &fakeEgress{name: "eg1", filters: map[string]struct{}{}, acceptAllHeads: true} + idx.Attach(eg) + + const sources = 8 + ev := StreamEvent{ + Kind: KindNewHead, + NetworkId: "evm:1", + Block: BlockRef{Number: 100, Hash: "0xAAA"}, + } + + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < sources; i++ { + wg.Add(1) + ev := ev + ev.SourceId = fmt.Sprintf("ws:up%d", i) + go func() { + defer wg.Done() + <-start + idx.Ingest(ev) + }() + } + close(start) + wg.Wait() + + if got := eg.count(); got != 1 { + t.Fatalf("concurrent ingest of identical head must dedupe to 1 delivery, got %d", got) + } +} + +// A newer head from one source must not overtake an older head another +// source is still delivering. +func TestIndexer_NewHead_ConcurrentSourcesDeliverInOrder(t *testing.T) { + idx := newIndexer(t) + idx.RegisterNetwork(newFakeNetwork("evm:1")) + delivering, release := make(chan struct{}), make(chan struct{}) + eg := &fakeEgress{name: "eg1", acceptAllHeads: true, hook: func(ev IndexedEvent) { + if ev.Block.Number == 100 { + close(delivering) + <-release + } + }} + idx.Attach(eg) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:up1", Block: BlockRef{Number: 100, Hash: "0xA"}}) + }() + <-delivering + go func() { + defer wg.Done() + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:up2", Block: BlockRef{Number: 101, Hash: "0xB"}}) + }() + time.Sleep(20 * time.Millisecond) // let the second head reach the indexer + close(release) + wg.Wait() + + eg.mu.Lock() + defer eg.mu.Unlock() + if len(eg.received) != 2 || eg.received[0].Block.Number != 100 || eg.received[1].Block.Number != 101 { + t.Fatalf("heads must be delivered in order, got %+v", eg.received) + } +} + +func TestIndexer_NewHead_DedupIgnoresHashCase(t *testing.T) { + idx := newIndexer(t) + idx.RegisterNetwork(newFakeNetwork("evm:1")) + eg := &fakeEgress{name: "eg1", acceptAllHeads: true} + idx.Attach(eg) + + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:up1", Block: BlockRef{Number: 100, Hash: "0xABC"}}) + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:up2", Block: BlockRef{Number: 100, Hash: "0xabc"}}) + if got := eg.count(); got != 1 { + t.Fatalf("same head with differently-cased hash must dedupe, got %d", got) + } +} + +func TestIndexer_NewHead_StalerDroppedKeepsStatePollerFed(t *testing.T) { + idx := newIndexer(t) + nw := newFakeNetwork("evm:1") + idx.RegisterNetwork(nw) + eg := &fakeEgress{name: "eg1", acceptAllHeads: true} + idx.Attach(eg) + + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:up1", Block: BlockRef{Number: 100, Hash: "0xAAA"}}) + // A sibling source sees a newer head; different source, same block — + // state poller must still receive the update even though fan-out dedupes. + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:up2", Block: BlockRef{Number: 100, Hash: "0xAAA"}}) + + if got := eg.count(); got != 1 { + t.Fatalf("duplicate head fan-out: want 1, got %d", got) + } + nw.mu.Lock() + defer nw.mu.Unlock() + if len(nw.suggestedBy["ws:up1"]) != 1 || len(nw.suggestedBy["ws:up2"]) != 1 { + t.Fatalf("each source must see its own SuggestLatestBlock, got %v", nw.suggestedBy) + } +} + +func TestIndexer_Log_RefcountFanOutAndTeardown(t *testing.T) { + idx := newIndexer(t) + nw := newFakeNetwork("evm:1") + idx.RegisterNetwork(nw) + ing := &fakeIngress{name: "ws:up1"} + if err := idx.AddIngress(context.Background(), "evm:1", ing); err != nil { + t.Fatal(err) + } + + params := []interface{}{"logs", map[string]interface{}{"topics": []string{"0x1"}}} + h1, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", params) + if err != nil { + t.Fatal(err) + } + // Second subscriber on same filter: refcount bumps, no new ingress call. + h2, _ := idx.EnsureFilter(context.Background(), "evm:1", "logs", params) + if h1 != h2 { + t.Fatalf("same params must hash identically, got %q vs %q", h1, h2) + } + if got := ing.ensureCalls.Load(); got != 1 { + t.Fatalf("EnsureFilter on ingress should be called exactly once, got %d", got) + } + + // First release decrements; still refcnt 1 → no tear-down. + idx.ReleaseFilter(context.Background(), "evm:1", "logs", h1) + if got := ing.removeCalls.Load(); got != 0 { + t.Fatalf("RemoveFilter must not fire while refcnt > 0, got %d calls", got) + } + // Second release drops to 0 → tear-down. + idx.ReleaseFilter(context.Background(), "evm:1", "logs", h1) + if got := ing.removeCalls.Load(); got != 1 { + t.Fatalf("RemoveFilter must fire exactly once at refcnt 0, got %d", got) + } +} + +// newLogFilter registers "evm:1", ensures an empty logs filter and attaches +// an egress interested in it. Helper for log dedup tests. +func newLogFilter(t *testing.T) (*Indexer, string, *fakeEgress) { + t.Helper() + idx := newIndexer(t) + idx.RegisterNetwork(newFakeNetwork("evm:1")) + if err := idx.AddIngress(context.Background(), "evm:1", &fakeIngress{name: "ws:up1"}); err != nil { + t.Fatal(err) + } + h, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs", map[string]interface{}{}}) + if err != nil { + t.Fatal(err) + } + eg := &fakeEgress{name: "eg1", filters: map[string]struct{}{h: {}}} + idx.Attach(eg) + return idx, h, eg +} + +func logEvent(filterHash, source, payload string) StreamEvent { + return StreamEvent{ + Kind: KindLog, NetworkId: "evm:1", SourceId: source, FilterHash: filterHash, + Payload: json.RawMessage(payload), + } +} + +const ( + logAddedJSON = `{"blockHash":"0xB","transactionHash":"0xT","logIndex":"0x0","removed":false}` + logRemovedJSON = `{"blockHash":"0xB","transactionHash":"0xT","logIndex":"0x0","removed":true}` +) + +func TestIndexer_Log_DuplicateAddsFromManyUpstreamsDeliverOnce(t *testing.T) { + idx, h, eg := newLogFilter(t) + for i := 0; i < 4; i++ { + idx.Ingest(logEvent(h, fmt.Sprintf("ws:up%d", i), logAddedJSON)) + } + if got := eg.count(); got != 1 { + t.Fatalf("same log from N upstreams must deliver once, got %d", got) + } +} + +// Reorg A→B→A where the log's block comes back with the same hash: the +// client must see add, remove, add — the final re-add must not be +// swallowed as a duplicate of the first add. +func TestIndexer_Log_AddRemoveReAddDeliversEach(t *testing.T) { + idx, h, eg := newLogFilter(t) + for _, p := range []string{logAddedJSON, logRemovedJSON, logAddedJSON} { + // Two upstreams each report every transition. + idx.Ingest(logEvent(h, "ws:up1", p)) + idx.Ingest(logEvent(h, "ws:up2", p)) + } + eg.mu.Lock() + defer eg.mu.Unlock() + if len(eg.received) != 3 { + t.Fatalf("add, remove, re-add must deliver 3 events, got %d", len(eg.received)) + } + want := []bool{false, true, false} + for i, ev := range eg.received { + if ev.Removed != want[i] { + t.Fatalf("event %d: Removed=%v, want %v", i, ev.Removed, want[i]) + } + } +} + +func TestIndexer_Log_DuplicateRemovesDeliverOnce(t *testing.T) { + idx, h, eg := newLogFilter(t) + idx.Ingest(logEvent(h, "ws:up1", logAddedJSON)) + idx.Ingest(logEvent(h, "ws:up1", logRemovedJSON)) + idx.Ingest(logEvent(h, "ws:up2", logRemovedJSON)) + idx.Ingest(logEvent(h, "ws:up3", logRemovedJSON)) + if got := eg.count(); got != 2 { + t.Fatalf("duplicate removes must collapse: want 2 deliveries, got %d", got) + } +} + +// Logs whose identity can't be established must pass through rather than +// collapse onto one shared key. +func TestIndexer_Log_MissingIdentityFieldsPassThrough(t *testing.T) { + idx, h, eg := newLogFilter(t) + idx.Ingest(logEvent(h, "ws:up1", `{"data":"0x01"}`)) + idx.Ingest(logEvent(h, "ws:up1", `{"data":"0x02"}`)) + idx.Ingest(logEvent(h, "ws:up1", `{"blockHash":"0xB","logIndex":"0x0"}`)) + if got := eg.count(); got != 3 { + t.Fatalf("logs without full identity must all pass through, got %d", got) + } +} + +// Hex on the wire is case-insensitive: the same log reported with +// differently-cased hashes by two upstreams is one log. The delivered +// payload is untouched. +func TestIndexer_Log_DedupIgnoresHexCase(t *testing.T) { + idx, h, eg := newLogFilter(t) + upper := `{"blockHash":"0xABCD","transactionHash":"0xEF01","logIndex":"0xA"}` + idx.Ingest(logEvent(h, "ws:up1", upper)) + idx.Ingest(logEvent(h, "ws:up2", `{"blockHash":"0xabcd","transactionHash":"0xef01","logIndex":"0xa"}`)) + eg.mu.Lock() + defer eg.mu.Unlock() + if len(eg.received) != 1 { + t.Fatalf("case-only differences must dedupe, got %d deliveries", len(eg.received)) + } + if string(eg.received[0].Payload) != upper { + t.Fatalf("payload must pass through verbatim, got %s", eg.received[0].Payload) + } +} + +// A subscriber arriving while the last one's teardown is still talking to +// the ingress must end up subscribed, not torn down by the late removal. +func TestIndexer_Filter_EnsureDuringReleaseStaysSubscribed(t *testing.T) { + ctx := context.Background() + idx := newIndexer(t) + idx.RegisterNetwork(newFakeNetwork("evm:1")) + removing, unblock := make(chan struct{}), make(chan struct{}) + ing := &fakeIngress{name: "a", hook: func(op string) { + if op == "remove" { + close(removing) + <-unblock + } + }} + if err := idx.AddIngress(ctx, "evm:1", ing); err != nil { + t.Fatal(err) + } + params := []interface{}{"logs"} + h, err := idx.EnsureFilter(ctx, "evm:1", "logs", params) + if err != nil { + t.Fatal(err) + } + + released := make(chan struct{}) + go func() { + idx.ReleaseFilter(ctx, "evm:1", "logs", h) + close(released) + }() + <-removing + ensured := make(chan error, 1) + go func() { + _, err := idx.EnsureFilter(ctx, "evm:1", "logs", params) + ensured <- err + }() + time.Sleep(20 * time.Millisecond) // let the new subscriber reach the indexer + close(unblock) + <-released + if err := <-ensured; err != nil { + t.Fatal(err) + } + if !ing.active.Load() { + t.Fatal("subscriber that arrived during teardown must end up subscribed") + } +} + +// A subscriber that joins while the first subscribe is still in flight must +// not be handed a subscription when that subscribe fails, and the failure +// must not leave a refcount behind that makes later subscribers skip the +// ingress. +func TestIndexer_Filter_FailedEnsureIsNotShared(t *testing.T) { + ctx := context.Background() + idx := newIndexer(t) + idx.RegisterNetwork(newFakeNetwork("evm:1")) + ensuring, unblock := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + ing := &fakeIngress{name: "a", hook: func(op string) { + if op == "ensure" && calls.Add(1) == 1 { + close(ensuring) + <-unblock + } + }} + ing.setErr(errTest("down")) + if err := idx.AddIngress(ctx, "evm:1", ing); err != nil { + t.Fatal(err) + } + params := []interface{}{"logs"} + + first, second := make(chan error, 1), make(chan error, 1) + go func() { + _, err := idx.EnsureFilter(ctx, "evm:1", "logs", params) + first <- err + }() + <-ensuring + go func() { + _, err := idx.EnsureFilter(ctx, "evm:1", "logs", params) + second <- err + }() + time.Sleep(20 * time.Millisecond) // let the second subscriber reach the indexer + close(unblock) + if <-first == nil { + t.Fatal("first subscribe must fail") + } + if <-second == nil { + t.Fatal("joining a failed subscribe must not return a silent subscription") + } + + ing.setErr(nil) + before := ing.ensureCalls.Load() + if _, err := idx.EnsureFilter(ctx, "evm:1", "logs", params); err != nil { + t.Fatal(err) + } + if ing.ensureCalls.Load() != before+1 { + t.Fatal("after a failed subscribe the next subscriber must subscribe on the ingress") + } +} + +func TestIndexer_Filter_ConcurrentEnsureReleaseConverges(t *testing.T) { + ctx := context.Background() + idx := newIndexer(t) + idx.RegisterNetwork(newFakeNetwork("evm:1")) + ing := &fakeIngress{name: "a"} + if err := idx.AddIngress(ctx, "evm:1", ing); err != nil { + t.Fatal(err) + } + params := []interface{}{"logs"} + + var wg sync.WaitGroup + for n := 0; n < 50; n++ { + wg.Add(1) + go func() { + defer wg.Done() + h, err := idx.EnsureFilter(ctx, "evm:1", "logs", params) + if err != nil { + t.Error(err) + return + } + idx.ReleaseFilter(ctx, "evm:1", "logs", h) + }() + } + wg.Wait() + if ing.active.Load() { + t.Fatal("every client released; the filter must be torn down") + } + if _, err := idx.EnsureFilter(ctx, "evm:1", "logs", params); err != nil { + t.Fatal(err) + } + if !ing.active.Load() { + t.Fatal("a new client must subscribe the filter again") + } +} + +// An ingress may keep a filter whose subscribe failed (to retry on +// reconnect). A failed EnsureFilter must remove it again. +func TestIndexer_Filter_FailedEnsureRemovesFromIngress(t *testing.T) { + idx := newIndexer(t) + a, b, _ := registerThreeIngresses(t, idx) + a.setErr(errTest("a")) + b.setErr(errTest("b")) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{defaults: []string{"a"}, fallbacks: []string{"b"}}) + + if _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}); err == nil { + t.Fatal("expected error") + } + if a.active.Load() || b.active.Load() { + t.Fatalf("failed subscribe left filters behind: a=%v b=%v", a.active.Load(), b.active.Load()) + } +} + +func TestIndexer_UnregisteredNetwork(t *testing.T) { + idx := newIndexer(t) + if err := idx.AddIngress(context.Background(), "evm:missing", &fakeIngress{name: "x"}); err == nil { + t.Fatal("expected error on unregistered network") + } + if _, err := idx.EnsureFilter(context.Background(), "evm:missing", "logs", []interface{}{"logs"}); err == nil { + t.Fatal("expected error on unregistered network") + } +} + +// registerThreeIngresses attaches three fakeIngresses named a, b, c on +// "evm:1" and returns them. Helper for selector tests. +func registerThreeIngresses(t *testing.T, idx *Indexer) (a, b, c *fakeIngress) { + t.Helper() + idx.RegisterNetwork(newFakeNetwork("evm:1")) + a = &fakeIngress{name: "a"} + b = &fakeIngress{name: "b"} + c = &fakeIngress{name: "c"} + for _, ing := range []*fakeIngress{a, b, c} { + if err := idx.AddIngress(context.Background(), "evm:1", ing); err != nil { + t.Fatal(err) + } + } + return +} + +func TestIndexer_Selector_DefaultsOnlyHappyPath(t *testing.T) { + idx := newIndexer(t) + a, b, c := registerThreeIngresses(t, idx) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{defaults: []string{"a", "b"}, fallbacks: []string{"c"}}) + + _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if a.ensureCalls.Load() != 1 || b.ensureCalls.Load() != 1 { + t.Fatalf("defaults must be called once each, got a=%d b=%d", a.ensureCalls.Load(), b.ensureCalls.Load()) + } + if c.ensureCalls.Load() != 0 { + t.Fatalf("fallback must not be touched when defaults succeed, got %d calls", c.ensureCalls.Load()) + } +} + +func TestIndexer_Selector_PartialDefaultFailureReturnsNil(t *testing.T) { + idx := newIndexer(t) + a, _, c := registerThreeIngresses(t, idx) + a.setErr(errTest("a is sad")) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{defaults: []string{"a", "b"}, fallbacks: []string{"c"}}) + + _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}) + if err != nil { + t.Fatalf("partial failure must not surface: %v", err) + } + if a.ensureCalls.Load() != 1 { + t.Fatalf("failing default must still be attempted once, got %d", a.ensureCalls.Load()) + } + if c.ensureCalls.Load() != 0 { + t.Fatalf("fallback must not be touched when at least one default succeeded, got %d", c.ensureCalls.Load()) + } +} + +func TestIndexer_Selector_AllDefaultsFailEscalatesToFallback(t *testing.T) { + idx := newIndexer(t) + a, b, c := registerThreeIngresses(t, idx) + a.setErr(errTest("a")) + b.setErr(errTest("b")) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{defaults: []string{"a", "b"}, fallbacks: []string{"c"}}) + + _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}) + if err != nil { + t.Fatalf("escalation should return nil when fallback succeeds: %v", err) + } + if c.ensureCalls.Load() != 1 { + t.Fatalf("fallback must be tried after all defaults fail, got %d calls", c.ensureCalls.Load()) + } +} + +// The selector excludes an ingress (e.g. a down upstream) by not naming it. +// An unnamed ingress must not be subscribed, and must not mask a failed +// default tier. +func TestIndexer_Selector_UnnamedIngressIsExcluded(t *testing.T) { + idx := newIndexer(t) + a, b, c := registerThreeIngresses(t, idx) + a.setErr(errTest("a")) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{defaults: []string{"a"}, fallbacks: []string{"b"}}) + + if _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}); err != nil { + t.Fatalf("fallback should have carried the filter: %v", err) + } + if c.ensureCalls.Load() != 0 { + t.Fatalf("unnamed ingress must not be subscribed, got %d calls", c.ensureCalls.Load()) + } + if b.ensureCalls.Load() != 1 { + t.Fatalf("fallback must be tried when every default failed, got %d calls", b.ensureCalls.Load()) + } +} + +func TestIndexer_Selector_NothingSelectedFails(t *testing.T) { + idx := newIndexer(t) + registerThreeIngresses(t, idx) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{}) + + if _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}); err == nil { + t.Fatal("a filter with no ingress behind it must fail, not return a silent subscription") + } +} + +func TestIndexer_Selector_AllFailReturnsJoinedError(t *testing.T) { + idx := newIndexer(t) + a, b, c := registerThreeIngresses(t, idx) + a.setErr(errTest("a")) + b.setErr(errTest("b")) + c.setErr(errTest("c")) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{defaults: []string{"a", "b"}, fallbacks: []string{"c"}}) + + _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}) + if err == nil { + t.Fatal("expected joined error when every tier fails") + } + msg := err.Error() + for _, want := range []string{"a", "b", "c"} { + if !containsStr(msg, want) { + t.Fatalf("joined error must include %q, got %q", want, msg) + } + } + // Re-attempt after fixing the ingresses must work — dedup window and + // refcount must have been rolled back. + a.setErr(nil) + b.setErr(nil) + c.setErr(nil) + if _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}); err != nil { + t.Fatalf("second attempt must succeed after errors clear, got %v", err) + } +} + +func TestIndexer_NilSelector_FansOutToAllIngresses(t *testing.T) { + idx := newIndexer(t) + a, b, c := registerThreeIngresses(t, idx) + // No selector registered: every ingress must be tried. + + h, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, ing := range []*fakeIngress{a, b, c} { + if ing.ensureCalls.Load() != 1 { + t.Fatalf("ingress %q: want 1 EnsureFilter call, got %d", ing.name, ing.ensureCalls.Load()) + } + } + idx.ReleaseFilter(context.Background(), "evm:1", "logs", h) + for _, ing := range []*fakeIngress{a, b, c} { + if ing.removeCalls.Load() != 1 { + t.Fatalf("ingress %q: want 1 RemoveFilter call, got %d", ing.name, ing.removeCalls.Load()) + } + } +} + +// Test-only helpers. + +type errTest string + +func (e errTest) Error() string { return string(e) } + +func containsStr(haystack, needle string) bool { + return len(needle) == 0 || indexOf(haystack, needle) >= 0 +} + +func indexOf(s, substr string) int { + n := len(substr) + if n == 0 { + return 0 + } + for i := 0; i+n <= len(s); i++ { + if s[i:i+n] == substr { + return i + } + } + return -1 +} diff --git a/indexer/ingress.go b/indexer/ingress.go new file mode 100644 index 000000000..3df2d4086 --- /dev/null +++ b/indexer/ingress.go @@ -0,0 +1,42 @@ +package indexer + +import "context" + +// Sink receives the StreamEvents an ingress produces; the Indexer is the +// Sink. Duplicates are expected and handled downstream. +type Sink interface { + Ingest(ev StreamEvent) +} + +// NetworkHandle is the part of a network an ingress may touch. +type NetworkHandle interface { + Id() string + // SuggestLatestBlock reports a head observed by source. It is called + // before dedup and fan-out, so every source's observation counts and + // the network tip moves before clients see the head. + SuggestLatestBlock(sourceId string, blockNumber int64) +} + +// EventIngress turns a transport-specific subscription into StreamEvents +// pushed at a Sink. +type EventIngress interface { + // Name identifies the ingress and must be stable for its lifetime. + Name() string + // Start begins pumping events and returns once the background workers + // run, not once the first event arrives. + Start(ctx context.Context, nw NetworkHandle, sink Sink) error + // EnsureFilter subscribes the filter with paramsHash (the + // BuildParamsKey of params); a no-op if it is already subscribed. + EnsureFilter(ctx context.Context, subType string, paramsHash string, params []interface{}) error + // RemoveFilter unsubscribes the filter; a no-op if it was never + // subscribed. Called when its last client leaves or its subscribe failed. + RemoveFilter(ctx context.Context, subType string, paramsHash string) error +} + +// IngressSelector chooses which ingresses carry a filter subscription, by +// Name(). Defaults are tried first and at least one must succeed; fallbacks +// are tried only if every default failed. Ingresses named in neither are +// excluded. Without a selector every ingress is a default. +type IngressSelector interface { + Select(networkId, subType string, params []interface{}) (defaults, fallbacks []string) +} diff --git a/indexer/integration_test.go b/indexer/integration_test.go new file mode 100644 index 000000000..976144854 --- /dev/null +++ b/indexer/integration_test.go @@ -0,0 +1,109 @@ +package indexer_test + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/erpc/erpc/indexer" + "github.com/erpc/erpc/indexer/adapters/nullingress" + "github.com/rs/zerolog" +) + +// TestIntegration_NullIngress_EndToEnd drives ingest, dedup and fan-out +// through a transport-free ingress. +func TestIntegration_NullIngress_EndToEnd(t *testing.T) { + logger := zerolog.New(zerolog.NewTestWriter(t)) + idx := indexer.New(&logger, indexer.Options{}) + + nw := &stubNetwork{id: "evm:1"} + idx.RegisterNetwork(nw) + + ing := nullingress.New("null:test") + if err := idx.AddIngress(context.Background(), "evm:1", ing); err != nil { + t.Fatalf("AddIngress: %v", err) + } + + // Egress that records newHeads. + eg := &recordingEgress{interested: func(k indexer.EventKind, _, _ string) bool { + return k == indexer.KindNewHead + }} + idx.Attach(eg) + + // Push two distinct heads and a dup — expect 2 deliveries. + ing.Push(headEvent("evm:1", "null:test", 100, "0xA", "0x0")) + ing.Push(headEvent("evm:1", "null:test", 100, "0xA", "0x0")) // dup + ing.Push(headEvent("evm:1", "null:test", 101, "0xB", "0xA")) + + deadline := time.After(2 * time.Second) + for { + if eg.count() >= 2 { + break + } + select { + case <-deadline: + t.Fatalf("timed out waiting for 2 deliveries, got %d", eg.count()) + case <-time.After(10 * time.Millisecond): + } + } + if got := eg.count(); got != 2 { + t.Fatalf("want 2 deliveries (dup suppressed), got %d", got) + } + nw.mu.Lock() + defer nw.mu.Unlock() + if n := len(nw.suggestions["null:test"]); n < 2 { + t.Fatalf("SuggestLatestBlock should fire on every observation (incl. dup), got %d", n) + } +} + +type stubNetwork struct { + id string + + mu sync.Mutex + suggestions map[string][]int64 +} + +func (s *stubNetwork) Id() string { return s.id } +func (s *stubNetwork) SuggestLatestBlock(sourceID string, block int64) { + s.mu.Lock() + if s.suggestions == nil { + s.suggestions = make(map[string][]int64) + } + s.suggestions[sourceID] = append(s.suggestions[sourceID], block) + s.mu.Unlock() +} + +type recordingEgress struct { + interested func(indexer.EventKind, string, string) bool + + mu sync.Mutex + recv []indexer.IndexedEvent +} + +func (r *recordingEgress) Name() string { return "test:recording" } +func (r *recordingEgress) InterestedIn(k indexer.EventKind, networkID, filterHash string) bool { + return r.interested(k, networkID, filterHash) +} +func (r *recordingEgress) Deliver(ev indexer.IndexedEvent) { + r.mu.Lock() + r.recv = append(r.recv, ev) + r.mu.Unlock() +} +func (r *recordingEgress) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.recv) +} + +func headEvent(networkID, sourceID string, num int64, hash, parent string) indexer.StreamEvent { + payload := json.RawMessage(`{"number":"0x0","hash":"` + hash + `","parentHash":"` + parent + `"}`) + return indexer.StreamEvent{ + Kind: indexer.KindNewHead, + NetworkId: networkID, + SourceId: sourceID, + Block: indexer.BlockRef{Number: num, Hash: hash}, + Payload: payload, + } +} From ea533ffb2293029456eafb9c6808c4c4f7dbbf7a Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 13:28:38 +0100 Subject: [PATCH 07/21] feat(ws): WebSocket JSON-RPC server with eth_subscribe HTTP and WebSocket share the port and URL paths. Over WebSocket: - single and batch requests run through the same pipeline as HTTP: forward headers, method filters, auth, directives with allowClientDirectives, rate limits and server.maxTimeout, with a per-connection in-flight limit; - eth_subscribe/eth_unsubscribe for newHeads, logs and newPendingTransactions via the indexer, with connection-scoped subscription ids and a per-connection subscription limit; - a slow reader drops its own oldest newHeads/pending-tx notifications; a logs subscription that overflows closes the connection with 1013; - peers silent for two ping intervals are dropped; on shutdown clients are closed with 1001 immediately while in-flight requests finish. Only a real RFC 6455 handshake bypasses gzip and maxTimeout. The method-not-allowed response now echoes the request id. Every message authenticates with the upgrade request's credentials, so an upgrade no auth strategy accepts is refused with 401; method scoping and rate limits still apply per message. A valid notification (no id) runs without a reply, alone or in a batch. A request that panics gets an error carrying its own id, and errors raised before network resolution use the URL's architecture for their wire code. Co-authored-by: snowkide Co-authored-by: shpookas Co-Authored-By: Claude Opus 5.5 --- auth/registry.go | 37 +- common/json_rpc.go | 22 +- erpc/admin_cordon_e2e_test.go | 2 +- erpc/grpc_server_test.go | 2 +- erpc/http_server.go | 271 ++-- erpc/http_server_directives_test.go | 72 ++ erpc/http_server_h2c_test.go | 6 +- erpc/http_server_logs_test.go | 12 +- erpc/http_server_test.go | 58 +- erpc/http_server_ws_tip_leader_test.go | 214 ++++ erpc/http_timeout.go | 8 + erpc/init.go | 2 +- erpc/networks_ws_tip_test.go | 207 ++++ erpc/projects.go | 9 + erpc/subscription_manager.go | 459 +++++++ erpc/ws_server.go | 625 ++++++++++ erpc/ws_server_hardening_test.go | 538 ++++++++ erpc/ws_server_selfheal_test.go | 234 ++++ erpc/ws_server_test.go | 1562 ++++++++++++++++++++++++ 19 files changed, 4243 insertions(+), 97 deletions(-) create mode 100644 erpc/http_server_ws_tip_leader_test.go create mode 100644 erpc/networks_ws_tip_test.go create mode 100644 erpc/subscription_manager.go create mode 100644 erpc/ws_server.go create mode 100644 erpc/ws_server_hardening_test.go create mode 100644 erpc/ws_server_selfheal_test.go create mode 100644 erpc/ws_server_test.go diff --git a/auth/registry.go b/auth/registry.go index 3d84429c2..a42d5aca4 100644 --- a/auth/registry.go +++ b/auth/registry.go @@ -101,16 +101,47 @@ func (r *AuthRegistry) Authenticate(ctx context.Context, req *common.NormalizedR return user, nil } + return nil, unauthorized(errs) +} + +// AcceptsPayload reports whether any strategy authenticates ap, whatever the +// method: nil when one does, or the error Authenticate would return. It +// applies no method scoping and takes no rate-limit permit, so it only +// rejects credentials no request could use; Authenticate still runs for +// every request. +func (r *AuthRegistry) AcceptsPayload(ctx context.Context, req *common.NormalizedRequest, ap *AuthPayload) error { + if ap == nil { + return common.NewErrAuthUnauthorized("n/a", "auth payload is nil") + } + if len(r.strategies) == 0 { + return nil + } + var errs []error + for _, az := range r.strategies { + if !az.strategy.Supports(ap) { + continue + } + if _, err := az.strategy.Authenticate(ctx, req, ap); err != nil { + errs = append(errs, err) + continue + } + return nil + } + return unauthorized(errs) +} + +// unauthorized is the error for a payload no strategy accepted. +func unauthorized(errs []error) error { if len(errs) == 1 { - return nil, errs[0] + return errs[0] } if len(errs) == 0 { - return nil, common.NewErrAuthUnauthorized("n/a", "no auth strategy matched make sure correct headers or query strings are provided") + return common.NewErrAuthUnauthorized("n/a", "no auth strategy matched make sure correct headers or query strings are provided") } // If no strategy matched or succeeded, consider the request unauthorized - return nil, common.NewErrAuthUnauthorized("n/a", errors.Join(errs...).Error()) + return common.NewErrAuthUnauthorized("n/a", errors.Join(errs...).Error()) } // FindDatabaseConnector finds a database connector by ID from the strategies diff --git a/common/json_rpc.go b/common/json_rpc.go index 636d46346..e3787fbe9 100644 --- a/common/json_rpc.go +++ b/common/json_rpc.go @@ -1216,6 +1216,10 @@ type JsonRpcRequest struct { // UnmarshalJSON. Empty when the request was constructed programmatically. idRaw []byte + // idAbsent records that the request had no "id" member: a notification + // (JSON-RPC 2.0 §4.1). "id": null is a request, not a notification. + idAbsent bool + // networkId holds the optional top-level "networkId" member, the body-side // equivalent of the // path segments. It is captured // by the envelope parse so routing reads it from the same result as @@ -1288,6 +1292,14 @@ func (r *JsonRpcRequest) RLockWithTrace(ctx context.Context) { r.RLock() } +// IsNotification reports whether the request had no "id" member, making it +// a JSON-RPC 2.0 notification that must not be answered. +func (r *JsonRpcRequest) IsNotification() bool { + r.RLock() + defer r.RUnlock() + return r.idAbsent +} + func (r *JsonRpcRequest) Clone() *JsonRpcRequest { r.RLock() defer r.RUnlock() @@ -1302,10 +1314,11 @@ func (r *JsonRpcRequest) Clone() *JsonRpcRequest { } clone := &JsonRpcRequest{ - JSONRPC: r.JSONRPC, - ID: r.ID, - Method: r.Method, - Params: clonedParams, + JSONRPC: r.JSONRPC, + ID: r.ID, + Method: r.Method, + Params: clonedParams, + idAbsent: r.idAbsent, } // Carry idRaw forward so the cloned request still round-trips its id // byte-for-byte through normalizeResponse. Without this, the clone falls @@ -1403,6 +1416,7 @@ func (r *JsonRpcRequest) UnmarshalJSON(data []byte) error { } r.Method = aux.Method.value r.networkId, _ = aux.NetworkID.(string) + r.idAbsent = aux.ID == nil if aux.ID != nil { var id interface{} diff --git a/erpc/admin_cordon_e2e_test.go b/erpc/admin_cordon_e2e_test.go index 596e5a825..c51c863bf 100644 --- a/erpc/admin_cordon_e2e_test.go +++ b/erpc/admin_cordon_e2e_test.go @@ -91,7 +91,7 @@ func TestAdminCordon_TwoInstancesConvergeThroughSharedState(t *testing.T) { require.NoError(t, err) instance.Bootstrap(ctx) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, instance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, instance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) diff --git a/erpc/grpc_server_test.go b/erpc/grpc_server_test.go index 02dcde9e2..578eaaab5 100644 --- a/erpc/grpc_server_test.go +++ b/erpc/grpc_server_test.go @@ -144,7 +144,7 @@ func TestHttpServer_CanSharePortWithGrpc(t *testing.T) { require.NoError(t, err) erpcInstance.Bootstrap(ctx) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) require.NotNil(t, httpServer.sharedGrpcServer) // Reflection is enabled by default, so the reflection service is registered diff --git a/erpc/http_server.go b/erpc/http_server.go index 65c27b48a..24d171111 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -23,8 +23,10 @@ import ( "github.com/bytedance/sonic" "github.com/erpc/erpc/auth" "github.com/erpc/erpc/common" + "github.com/erpc/erpc/indexer" "github.com/erpc/erpc/telemetry" "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" "github.com/klauspost/compress/gzip" "github.com/rs/zerolog" "go.opentelemetry.io/otel/attribute" @@ -54,6 +56,9 @@ type HttpServer struct { trustedForwarderIPs map[string]struct{} trustedIPHeaders []string resolvedResponseHeaders map[string]string + subscriptionManager *SubscriptionManager + activeWsConns sync.Map // connId -> *WsConnection + reqMaxTimeout time.Duration } func NewHttpServer( @@ -62,6 +67,7 @@ func NewHttpServer( cfg *common.ServerConfig, healthCheckCfg *common.HealthCheckConfig, adminCfg *common.AdminConfig, + indexerCfg *common.IndexerConfig, erpc *ERPC, ) (*HttpServer, error) { reqMaxTimeout := 150 * time.Second @@ -86,16 +92,26 @@ func NewHttpServer( gzipPool := util.NewGzipReaderPool() + subMgrLogger := logger.With().Str("component", "subscriptions").Logger() + indexerLogger := logger.With().Str("component", "indexer").Logger() + indexerOpts := indexer.Options{} + if indexerCfg != nil { + indexerOpts.DedupWindowSize = indexerCfg.DedupWindowSize + } + idx := indexer.New(&indexerLogger, indexerOpts) + srv := &HttpServer{ - logger: logger, - appCtx: ctx, - serverCfg: cfg, - healthCheckCfg: healthCheckCfg, - adminCfg: adminCfg, - erpc: erpc, - draining: &draining, - drained: make(chan struct{}), - gzipPool: gzipPool, + logger: logger, + appCtx: ctx, + serverCfg: cfg, + healthCheckCfg: healthCheckCfg, + adminCfg: adminCfg, + erpc: erpc, + draining: &draining, + drained: make(chan struct{}), + gzipPool: gzipPool, + subscriptionManager: NewSubscriptionManager(&subMgrLogger, idx), + reqMaxTimeout: reqMaxTimeout, } if cfg != nil { @@ -240,11 +256,17 @@ func NewHttpServer( if srv.serverV6 != nil { srv.serverV6.SetKeepAlivesEnabled(false) } - // wait for readiness probe to mark the pod NotReady - // ideally (period_seconds * failure_threshold) + safety margin (1s) + // Hijacked WebSocket connections are not affected by the above, so + // close them now: clients reconnect elsewhere while in-flight + // requests get the grace window to finish. + var grace time.Duration if srv.serverCfg.WaitBeforeShutdown != nil { - time.Sleep(srv.serverCfg.WaitBeforeShutdown.Duration()) + grace = srv.serverCfg.WaitBeforeShutdown.Duration() } + go srv.shutdownWebSockets(logger, grace) + // wait for readiness probe to mark the pod NotReady + // ideally (period_seconds * failure_threshold) + safety margin (1s) + time.Sleep(grace) // Shutdown closes the listeners and idle HTTP/1.1 connections, sends // GOAWAY on every HTTP/2 connection, and waits (within its budget) for // in-flight requests and streams to finish. @@ -384,6 +406,52 @@ func (s *HttpServer) createRequestHandler() http.Handler { } } + // A WebSocket connection serves the one network in its URL. + if isWebSocketUpgradeRequest(r) { + if architecture == "" || chainId == "" { + handleErrorResponse( + httpCtx, + &lg, + &startedAt, + nil, + common.NewErrInvalidUrlPath("websocket connections must target a network, for example ///", r.URL.Path), + w, + encoder, + writeFatalError, + &common.TRUE, + s.executionHeadersMode(), + ) + return + } + // Every message on the connection authenticates with this upgrade + // request's credentials, so refuse the upgrade when no strategy + // accepts them rather than hold a connection that can never be + // served. Method scoping and rate limits still apply per message. + nq := common.NewNormalizedRequest(nil) + nq.SetClientIP(s.resolveRealClientIP(r)) + ap, err := auth.NewPayloadFromHttp("", r.RemoteAddr, r.Header, r.URL.Query()) + if err == nil { + err = project.AcceptsConsumerPayload(httpCtx, nq, ap) + } + if err != nil { + handleErrorResponse( + httpCtx, + &lg, + &startedAt, + nil, + err, + w, + encoder, + writeFatalError, + s.serverCfg.IncludeErrorDetails, + s.executionHeadersMode(), + ) + return + } + s.handleWebSocket(w, r, &lg, project, architecture, chainId) + return + } + // Handle gzipped request bodies var bodyReader io.Reader = r.Body if r.Header.Get("Content-Encoding") == "gzip" { @@ -513,63 +581,26 @@ func (s *HttpServer) createRequestHandler() http.Handler { return } - if project != nil { - for _, matchKey := range project.Config.ForwardHeaders { - for key, values := range headers { - matches, err := common.WildcardMatch(matchKey, key) - if err != nil { - responses[index] = processErrorBody(&lg, &startedAt, nq, err, &common.TRUE) - common.EndRequestSpan(requestCtx, nil, responses[index]) - return - } - if matches { - for _, value := range values { - nq.ForwardHeaders.Add(matchKey, value) - } - } - } - } + if err := applyForwardHeaders(project, nq, headers); err != nil { + responses[index] = processErrorBody(&lg, &startedAt, nq, err, &common.TRUE) + common.EndRequestSpan(requestCtx, nil, responses[index]) + return } method, _ := nq.Method() rlg := lg.With().Str("method", method).Logger() - shouldHandleMethod := true - - if project != nil && project.Config.IgnoreMethods != nil { - for _, m := range project.Config.IgnoreMethods { - match, err := common.WildcardMatch(m, method) - if err != nil { - responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) - common.EndRequestSpan(requestCtx, nil, err) - return - } - if match { - shouldHandleMethod = false - break - } - } - } - - if project != nil && project.Config.AllowMethods != nil { - for _, m := range project.Config.AllowMethods { - match, err := common.WildcardMatch(m, method) - if err != nil { - responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) - common.EndRequestSpan(requestCtx, nil, err) - return - } - if match { - shouldHandleMethod = true - break - } - } + shouldHandleMethod, err := isMethodAllowed(project, method) + if err != nil { + responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) + common.EndRequestSpan(requestCtx, nil, err) + return } if !shouldHandleMethod { jsonrpcVersion := "2.0" var reqId interface{} - if jrr, err := nq.JsonRpcRequest(); err != nil { + if jrr, _ := nq.JsonRpcRequest(); jrr != nil { jsonrpcVersion = jrr.JSONRPC reqId = jrr.ID } @@ -588,7 +619,6 @@ func (s *HttpServer) createRequestHandler() http.Handler { } var ap *auth.AuthPayload - var err error if project != nil { ap, err = auth.NewPayloadFromHttp(method, r.RemoteAddr, headers, queryArgs) @@ -719,16 +749,7 @@ func (s *HttpServer) createRequestHandler() http.Handler { } nq.SetNetwork(nw) - nq.ApplyDirectiveDefaults(nw.Config().DirectiveDefaults) - // Configure how to store User-Agent (raw vs simplified) based on project config - uaMode := common.UserAgentTrackingModeSimplified - if project != nil { - if project.Config.UserAgentMode != "" { - uaMode = project.Config.UserAgentMode - } - nq.SetAllowClientDirectiveMatcher(project.clientDirectiveMatcherFor(nq.User())) - } - nq.EnrichFromHttp(headers, queryArgs, uaMode) + applyRequestDirectives(project, nw, nq, headers, queryArgs) rlg.Trace().Interface("directives", nq.Directives()).Msgf("applied request directives") resp, err := project.Forward(requestCtx, networkId, nq) @@ -1074,7 +1095,7 @@ func (s *HttpServer) parseUrlPath( return "", "", "", false, false, common.NewErrInvalidUrlPath("architecture is not valid (must be 'evm' or 'svm')", ps) } - if !isPost && !isOptions { + if !isPost && !isOptions && !isWebSocketUpgradeRequest(r) { isHealthCheck = true } @@ -2019,6 +2040,8 @@ func (s *HttpServer) createTLSConfig() (*tls.Config, error) { func (s *HttpServer) Shutdown(logger *zerolog.Logger) error { logger.Info().Msg("stopping http servers...") + s.shutdownWebSockets(logger, 0) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -2062,6 +2085,36 @@ func (s *HttpServer) Shutdown(logger *zerolog.Logger) error { return lastErr } +// shutdownWebSockets refuses new WebSocket connections and closes the active +// ones with GoingAway, giving their in-flight requests up to grace to +// finish, then waits for subscription cleanup to complete. +func (s *HttpServer) shutdownWebSockets(logger *zerolog.Logger, grace time.Duration) { + s.draining.Store(true) + + var conns []*WsConnection + s.activeWsConns.Range(func(_, value interface{}) bool { + wsc := value.(*WsConnection) + wsc.stop(websocket.CloseGoingAway, "server shutting down", grace) + conns = append(conns, wsc) + return true + }) + if len(conns) == 0 { + return + } + + logger.Info().Int("connections", len(conns)).Msg("closing active WebSocket connections...") + timeout := time.After(grace + 10*time.Second) + for _, wsc := range conns { + select { + case <-wsc.done: + case <-timeout: + logger.Warn().Int("connections", len(conns)).Msg("timed out waiting for WebSocket connections to close") + return + } + } + logger.Info().Int("connections", len(conns)).Msg("all WebSocket connections closed") +} + // conditionalGzipWriter wraps ResponseWriter and decides whether to compress // once enough body bytes have been observed. Writes (and any explicit status // code) are held back until the total body size reaches compressionThreshold, @@ -2191,6 +2244,14 @@ func gzipHandler(next http.Handler) http.Handler { var gzPool = util.NewGzipWriterPool() return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A WebSocket handshake has no body to compress and must reach the + // upgrader with a ResponseWriter that can be hijacked, which the + // gzip writer cannot. + if isWebSocketUpgradeRequest(r) { + next.ServeHTTP(w, r) + return + } + // The response representation depends on Accept-Encoding, so caches // must be told regardless of whether this response ends up compressed. w.Header().Set("Vary", "Accept-Encoding") @@ -2341,6 +2402,74 @@ func stripAddrDecorations(s string) string { return s } +// applyForwardHeaders copies the request headers matching the project's +// forwardHeaders patterns onto nq. +func applyForwardHeaders(project *PreparedProject, nq *common.NormalizedRequest, headers http.Header) error { + if project == nil { + return nil + } + for _, matchKey := range project.Config.ForwardHeaders { + for key, values := range headers { + matches, err := common.WildcardMatch(matchKey, key) + if err != nil { + return err + } + if matches { + for _, value := range values { + nq.ForwardHeaders.Add(matchKey, value) + } + } + } + } + return nil +} + +// isMethodAllowed applies the project's ignoreMethods / allowMethods +// (allowMethods wins). +func isMethodAllowed(project *PreparedProject, method string) (bool, error) { + if project == nil { + return true, nil + } + allowed := true + for _, m := range project.Config.IgnoreMethods { + match, err := common.WildcardMatch(m, method) + if err != nil { + return false, err + } + if match { + allowed = false + break + } + } + for _, m := range project.Config.AllowMethods { + match, err := common.WildcardMatch(m, method) + if err != nil { + return false, err + } + if match { + allowed = true + break + } + } + return allowed, nil +} + +// applyRequestDirectives applies network directive defaults and then the +// client's own directives from headers/query, subject to the project's +// allowClientDirectives for the resolved user. +func applyRequestDirectives(project *PreparedProject, nw *Network, nq *common.NormalizedRequest, headers http.Header, queryArgs map[string][]string) { + nq.ApplyDirectiveDefaults(nw.Config().DirectiveDefaults) + // Configure how to store User-Agent (raw vs simplified) based on project config + uaMode := common.UserAgentTrackingModeSimplified + if project != nil { + if project.Config.UserAgentMode != "" { + uaMode = project.Config.UserAgentMode + } + nq.SetAllowClientDirectiveMatcher(project.clientDirectiveMatcherFor(nq.User())) + } + nq.EnrichFromHttp(headers, queryArgs, uaMode) +} + // isAdminMethodBlocked returns true when the admin config's DenyMethods/AllowMethods // rules prevent the given method from being handled. // DenyMethods is evaluated first; AllowMethods can re-admit a method that was denied. diff --git a/erpc/http_server_directives_test.go b/erpc/http_server_directives_test.go index d05fca523..715cf377b 100644 --- a/erpc/http_server_directives_test.go +++ b/erpc/http_server_directives_test.go @@ -1,6 +1,7 @@ package erpc import ( + "context" "net/http" "strings" "testing" @@ -9,7 +10,9 @@ import ( "github.com/erpc/erpc/common" "github.com/erpc/erpc/internal/policy" "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" "github.com/h2non/gock" + "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -173,3 +176,72 @@ func TestHttpServer_AuthStrategyClientDirectives(t *testing.T) { assert.Contains(t, respBody, "0x2222222") }) } + +// TestWebSocket_AuthStrategyClientDirectives is the WebSocket counterpart of +// TestHttpServer_AuthStrategyClientDirectives: directives sent on the upgrade +// request obey the same allowClientDirectives rules, for single and batch +// requests alike. +func TestWebSocket_AuthStrategyClientDirectives(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + gock.New("http://rpc1.localhost"). + Post("/"). + Persist(). + Filter(func(request *http.Request) bool { + return strings.Contains(util.SafeReadBody(request), "eth_getBlockNumber") + }). + Reply(200). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 111, "result": "0x1111111"}) + gock.New("http://rpc2.localhost"). + Post("/"). + Persist(). + Filter(func(request *http.Request) bool { + return strings.Contains(util.SafeReadBody(request), "eth_getBlockNumber") + }). + Reply(200). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 222, "result": "0x2222222"}) + + cfg := authStrategyDirectivesCfg() + cfg.Server.ListenV4 = util.BoolPtr(true) + srv, addr, cancel := startTestERPCServer(t, cfg) + defer func() { _ = srv.Shutdown(&log.Logger); cancel() }() + + prj, err := srv.erpc.GetProject("test_project") + require.NoError(t, err) + _, err = prj.GetNetwork(context.Background(), "evm:123") + require.NoError(t, err) + policy.OverrideAllForTest(prj.policyEngine) + + send := func(t *testing.T, token, body string) string { + hdr := http.Header{} + hdr.Set("X-ERPC-Secret-Token", token) + hdr.Set("X-ERPC-Use-Upstream", "rpc2") + conn, _, err := websocket.DefaultDialer.Dial("ws://"+addr+"/test_project/evm/123", hdr) + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(body))) + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err) + return string(msg) + } + + const single = `{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}` + const batch = `[` + single + `]` + + // Deny cases first, so rpc2 has served no traffic when they select. + t.Run("SingleDeniedDirectiveIsDropped", func(t *testing.T) { + assert.Contains(t, send(t, authDirectivesCustomerToken, single), "0x1111111") + }) + t.Run("BatchDeniedDirectiveIsDropped", func(t *testing.T) { + assert.Contains(t, send(t, authDirectivesCustomerToken, batch), "0x1111111") + }) + t.Run("SingleGrantedDirectiveApplies", func(t *testing.T) { + assert.Contains(t, send(t, authDirectivesOpsToken, single), "0x2222222") + }) + t.Run("BatchGrantedDirectiveApplies", func(t *testing.T) { + assert.Contains(t, send(t, authDirectivesOpsToken, batch), "0x2222222") + }) +} diff --git a/erpc/http_server_h2c_test.go b/erpc/http_server_h2c_test.go index 5825a49b0..c72793562 100644 --- a/erpc/http_server_h2c_test.go +++ b/erpc/http_server_h2c_test.go @@ -64,7 +64,7 @@ func TestHttpServer_H2C_WithoutSharedGrpc(t *testing.T) { require.NoError(t, err) erpcInstance.Bootstrap(ctx) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) require.Nil(t, httpServer.sharedGrpcServer, "precondition: shared gRPC must be disabled") @@ -212,7 +212,7 @@ func TestHttpServer_H2C_ShutdownGoAwaysAndDrainsInFlightStream(t *testing.T) { require.NoError(t, err) erpcInstance.Bootstrap(appCtx) - httpServer, err := NewHttpServer(appCtx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(appCtx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") @@ -364,7 +364,7 @@ func TestHttpServer_SharedGrpc_ShutdownDrainsInFlightRPC(t *testing.T) { require.NoError(t, err) erpcInstance.Bootstrap(appCtx) - httpServer, err := NewHttpServer(appCtx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(appCtx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/erpc/http_server_logs_test.go b/erpc/http_server_logs_test.go index 3a28a74ee..a2ea6935b 100644 --- a/erpc/http_server_logs_test.go +++ b/erpc/http_server_logs_test.go @@ -118,7 +118,7 @@ func TestHttp_EvmGetLogs_SplitOnError_MergedResponse(t *testing.T) { erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") @@ -263,7 +263,7 @@ func TestHttp_EvmGetLogs_ProactiveRangeSplit_MergedResponse(t *testing.T) { erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") @@ -491,7 +491,7 @@ func TestHttp_EvmGetLogs_SplitOnError_ByAddresses_MergedResponse(t *testing.T) { require.NoError(t, err) erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) @@ -589,7 +589,7 @@ func TestHttp_EvmGetLogs_SplitOnError_ByTopic0ORList_MergedResponse(t *testing.T require.NoError(t, err) erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) @@ -675,7 +675,7 @@ func TestHttp_EvmGetLogs_SplitOnError_EmptyAndNonEmptyMergedSkipsEmpty(t *testin require.NoError(t, err) erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) @@ -743,7 +743,7 @@ func TestHttp_ConcurrentIdenticalRequests_NoEmptyBodyParse(t *testing.T) { require.NoError(t, err) erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/erpc/http_server_test.go b/erpc/http_server_test.go index cdd916001..ba47c0a00 100644 --- a/erpc/http_server_test.go +++ b/erpc/http_server_test.go @@ -112,7 +112,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) // Start the server on a random port @@ -254,7 +254,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) // Start the server on a random port @@ -400,7 +400,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) // Start the server on a random port @@ -7865,7 +7865,7 @@ func createServerTestFixtures(cfg *common.Config, t *testing.T) ( erpcInstance.Bootstrap(ctx) require.NoError(t, err) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") @@ -8157,7 +8157,7 @@ func TestHttpServer_Evm_GetLogs_MemoryProfile(t *testing.T) { // Give state poller more time to initialize and update shared state time.Sleep(1 * time.Second) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") @@ -8276,7 +8276,7 @@ func TestHttpServer_DrainStampsConnectionClose(t *testing.T) { require.NoError(t, err) erpcInstance.Bootstrap(ctx) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") @@ -8323,6 +8323,50 @@ func TestHttpServer_DrainStampsConnectionClose(t *testing.T) { require.True(t, resp.Close, "drain-window responses must carry Connection: close so pooled clients migrate before Shutdown") } +func TestHttpServer_IgnoredMethodKeepsRequestId(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + cfg := &common.Config{ + Server: &common.ServerConfig{ + MaxTimeout: common.Duration(5 * time.Second).Ptr(), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_project", + IgnoreMethods: []string{"eth_getBalance"}, + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } + + sendRequest, _, _, shutdown, _ := createServerTestFixtures(cfg, t) + defer shutdown() + + statusCode, _, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x123","latest"],"id":7}`, nil, nil) + assert.Equal(t, http.StatusOK, statusCode) + + var resp map[string]interface{} + require.NoError(t, json.Unmarshal([]byte(body), &resp)) + assert.Equal(t, float64(7), resp["id"], "the error response must echo the request id") + errMap, _ := resp["error"].(map[string]interface{}) + assert.Contains(t, errMap["message"], "method not supported") +} + func TestHttpServer_AdminMethodFilter(t *testing.T) { const adminSecret = "test-secret" @@ -8369,7 +8413,7 @@ func TestHttpServer_AdminMethodFilter(t *testing.T) { require.NoError(t, err) erpcInstance.Bootstrap(ctx) - httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) require.NoError(t, err) listener, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/erpc/http_server_ws_tip_leader_test.go b/erpc/http_server_ws_tip_leader_test.go new file mode 100644 index 000000000..8e1a8c497 --- /dev/null +++ b/erpc/http_server_ws_tip_leader_test.go @@ -0,0 +1,214 @@ +package erpc + +import ( + "context" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/bytedance/sonic" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/internal/policy" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { + util.ConfigureTestLogger() +} + +// setupTipLeaderServer starts a server with two upstreams, rpc1 ordered ahead +// of rpc2, and returns the network and rpc2. +func setupTipLeaderServer(t *testing.T, evmCfg *common.EvmNetworkConfig, maxAttempts int) ( + sendRequest func(body string, headers map[string]string, queryParams map[string]string) (int, map[string]string, string), + shutdown func(), + nw *Network, + leader *upstream.Upstream, +) { + t.Helper() + upstreams := make([]*common.UpstreamConfig, 0, 2) + for _, id := range []string{"rpc1", "rpc2"} { + upstreams = append(upstreams, &common.UpstreamConfig{ + Id: id, + Endpoint: "http://" + id + ".localhost", + Type: common.UpstreamTypeEvm, + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(10 * time.Second), + }, + }) + } + cfg := &common.Config{ + Server: &common.ServerConfig{ + MaxTimeout: common.Duration(100 * time.Second).Ptr(), + }, + Projects: []*common.ProjectConfig{{ + Id: "test_project", + Networks: []*common.NetworkConfig{{ + Architecture: "evm", + Evm: evmCfg, + Failsafe: []*common.FailsafeConfig{{ + Retry: &common.RetryPolicyConfig{MaxAttempts: maxAttempts}, + }}, + }}, + Upstreams: upstreams, + }}, + } + + sendRequest, _, _, shutdown, erpcInstance := createServerTestFixtures(cfg, t) + prj, err := erpcInstance.GetProject("test_project") + require.NoError(t, err) + policy.OverrideAllForTest(prj.policyEngine) + policy.OverrideOrderForTest(prj.policyEngine, "evm:123", "rpc1", "rpc2") + + time.Sleep(500 * time.Millisecond) + + nw, err = prj.GetNetwork(context.Background(), "evm:123") + require.NoError(t, err) + for _, u := range nw.upstreamsRegistry.GetNetworkUpstreams(context.Background(), "evm:123") { + if u.Id() == "rpc2" { + leader = u + } + } + require.NotNil(t, leader) + return sendRequest, shutdown, nw, leader +} + +// EnforceHighestBlock's tip re-fetch must reach the upstream whose poller has +// the tip, not settle for the lagging sibling that answered "latest" first. +func TestHttpServer_GetBlockByNumberLatest_RefetchReachesTipUpstream(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.AssertNoPendingMocks(t, 0) + + const tip = int64(0x22228889) + tipHex := "0x22228889" + var leaderHits atomic.Int64 + + gock.New("http://rpc2.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + if !strings.Contains(body, "eth_getBlockByNumber") || !strings.Contains(body, tipHex) { + return false + } + leaderHits.Add(1) + return true + }). + Reply(200). + JSON([]byte(`{"result":{"number":"0x22228889","hash":"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","parentHash":"0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","timestamp":"0x6702a8f1"}}`)) + + // rpc1 lags, so the initial "latest" comes from it and triggers the + // EnforceHighestBlock re-fetch. + sendRequest, shutdown, nw, leader := setupTipLeaderServer(t, &common.EvmNetworkConfig{ + ChainId: 123, + Integrity: &common.EvmIntegrityConfig{ + EnforceHighestBlock: util.BoolPtr(true), + }, + }, 3) + defer shutdown() + + // Mirror WS ingest: the tip-source poller and the delivered-head floor + // advance before any client sees the head. + leader.EvmStatePoller().SuggestLatestBlock(tip) + nw.NoteObservedLatestBlock(context.Background(), tip) + + require.Equal(t, tip, leader.EvmStatePoller().LatestBlock()) + require.Equal(t, "rpc2", nw.EvmLeaderUpstream(context.Background()).Id()) + require.Equal(t, tip, nw.EvmHighestLatestBlockNumber(context.Background())) + + statusCode, _, body := sendRequest(`{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_getBlockByNumber", + "params": ["latest", false] + }`, nil, nil) + + require.Equal(t, http.StatusOK, statusCode) + + var respObject map[string]interface{} + require.NoError(t, sonic.UnmarshalString(body, &respObject)) + result, ok := respObject["result"].(map[string]interface{}) + require.True(t, ok, "response should have a result object, got: %s", body) + assert.Equal(t, tipHex, result["number"]) + assert.Equal(t, "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", result["hash"]) + assert.GreaterOrEqual(t, leaderHits.Load(), int64(1), + "the tip re-fetch must reach the upstream that has the tip") +} + +// Direct eth_getBlockByNumber(tip) goes first to the upstream that has the +// tip, not to a lagging sibling preferred by selection order. +func TestHttpServer_GetBlockByNumber_NearTipPrefersTipUpstream(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + // rpc1 tip-null Persist mock is intentionally unused. + defer util.AssertNoPendingMocks(t, 1) + + const tip = int64(0x33338889) + tipHex := "0x33338889" + var leaderHits atomic.Int64 + var laggingHits atomic.Int64 + + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + if r.URL.Host != "rpc1.localhost" { + return false + } + body := util.SafeReadBody(r) + if strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, tipHex) { + laggingHits.Add(1) + return true + } + return false + }). + Reply(200). + JSON([]byte(`{"result":null}`)) + + gock.New("http://rpc2.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + if r.URL.Host != "rpc2.localhost" { + return false + } + body := util.SafeReadBody(r) + if strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, tipHex) { + leaderHits.Add(1) + return true + } + return false + }). + Reply(200). + JSON([]byte(`{"result":{"number":"0x33338889","hash":"0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","parentHash":"0xdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","timestamp":"0x6702a8f1"}}`)) + + sendRequest, shutdown, nw, leader := setupTipLeaderServer(t, &common.EvmNetworkConfig{ChainId: 123}, 2) + defer shutdown() + leader.EvmStatePoller().SuggestLatestBlock(tip) + require.Equal(t, "rpc2", nw.EvmLeaderUpstream(context.Background()).Id()) + + statusCode, _, body := sendRequest(`{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_getBlockByNumber", + "params": ["`+tipHex+`", false] + }`, nil, nil) + + require.Equal(t, http.StatusOK, statusCode, "body=%s", body) + var respObject map[string]interface{} + require.NoError(t, sonic.UnmarshalString(body, &respObject)) + result, ok := respObject["result"].(map[string]interface{}) + require.True(t, ok, "got: %s", body) + assert.Equal(t, tipHex, result["number"]) + assert.GreaterOrEqual(t, leaderHits.Load(), int64(1), + "near-tip getBlock must go to the upstream that has the tip") + assert.Equal(t, int64(0), laggingHits.Load(), + "lagging sibling must not receive the near-tip getBlock") +} diff --git a/erpc/http_timeout.go b/erpc/http_timeout.go index eaee7f918..9295de057 100644 --- a/erpc/http_timeout.go +++ b/erpc/http_timeout.go @@ -34,6 +34,14 @@ type timeoutHandler struct { } func (h *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // The timeout writer cannot be hijacked, which a WebSocket upgrade + // needs. Each request over the connection gets the same deadline in + // ws_server.go instead. + if isWebSocketUpgradeRequest(r) { + h.handler.ServeHTTP(w, r) + return + } + ctx, cancelCtx := context.WithTimeoutCause(r.Context(), h.dt, ErrHandlerTimeout) defer func() { cancelCtx() diff --git a/erpc/init.go b/erpc/init.go index 36aa37c8f..59dec553e 100644 --- a/erpc/init.go +++ b/erpc/init.go @@ -152,7 +152,7 @@ func Init( // cancelled; nil (never waited on) when no HTTP server is configured. var httpDrained <-chan struct{} if cfg.Server != nil { - httpServer, err := NewHttpServer(appCtx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + httpServer, err := NewHttpServer(appCtx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) if err != nil { return err } diff --git a/erpc/networks_ws_tip_test.go b/erpc/networks_ws_tip_test.go new file mode 100644 index 000000000..215da21d6 --- /dev/null +++ b/erpc/networks_ws_tip_test.go @@ -0,0 +1,207 @@ +package erpc + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/health" + "github.com/erpc/erpc/thirdparty" + "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { + util.ConfigureTestLogger() +} + +// setupWsTipNetwork bootstraps a single-upstream EVM network backed by an +// in-memory shared state registry. +func setupWsTipNetwork(t *testing.T, ctx context.Context, id, endpoint string) (*Network, *upstream.Upstream) { + t.Helper() + util.SetupMocksForEvmStatePoller() + gock.New(endpoint). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), `eth_chainId`) + }). + Reply(200). + JSON([]byte(`{"result":"0x7b"}`)) + + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) + metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) + vr := thirdparty.NewVendorsRegistry() + pr, err := thirdparty.NewProvidersRegistry(&log.Logger, vr, []*common.ProviderConfig{}, nil) + require.NoError(t, err) + ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}, + }, + }) + require.NoError(t, err) + + upstreamsRegistry := upstream.NewUpstreamsRegistry( + ctx, &log.Logger, "test", + []*common.UpstreamConfig{{ + Type: common.UpstreamTypeEvm, + Id: id, + Endpoint: endpoint, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }}, ssr, rateLimitersRegistry, vr, pr, nil, + metricsTracker, nil, + ) + network, err := NewNetwork(ctx, &log.Logger, "test", &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, rateLimitersRegistry, upstreamsRegistry, metricsTracker, nil) + require.NoError(t, err) + + upstreamsRegistry.Bootstrap(ctx) + time.Sleep(200 * time.Millisecond) + require.NoError(t, upstreamsRegistry.GetInitializer().WaitForTasks(ctx)) + require.NoError(t, network.Bootstrap(ctx)) + time.Sleep(250 * time.Millisecond) + + upsList := upstreamsRegistry.GetNetworkUpstreams(ctx, util.EvmNetworkId(123)) + require.Len(t, upsList, 1) + return network, upsList[0] +} + +// Once a newHeads head N is about to be delivered, "latest" must not return +// less than N even while every poller still reports N-1. +func TestNoteObservedLatestBlock_FloorsEvmHighestLatest(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, u := setupWsTipNetwork(t, ctx, "rpc1", "http://rpc1.localhost") + u.EvmStatePoller().SuggestLatestBlock(1000) + time.Sleep(50 * time.Millisecond) + require.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx)) + + network.NoteObservedLatestBlock(ctx, 1001) + + assert.Equal(t, int64(1001), network.EvmHighestLatestBlockNumber(ctx), + "after WS tip observation, highest latest must be ≥ delivered head") + require.NotNil(t, network.latestBlockShared) + assert.GreaterOrEqual(t, network.deliveredLatestBlock.Load(), int64(1001)) + assert.Equal(t, int64(1001), network.EvmHighestLatestBlockNumber(ctx)) +} + +// End-to-end through networkHandle.SuggestLatestBlock, which runs before +// fan-out. +func TestNetworkHandle_SuggestLatestBlock_AdvancesNetworkTipBeforeFanOut(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, u := setupWsTipNetwork(t, ctx, "rpc2", "http://rpc2.localhost") + u.EvmStatePoller().SuggestLatestBlock(90677358) + time.Sleep(50 * time.Millisecond) + require.Equal(t, int64(90677358), network.EvmHighestLatestBlockNumber(ctx)) + + handle := &networkHandle{nw: network} + handle.SuggestLatestBlock("ws:rpc2", 90677359) + + assert.Equal(t, int64(90677359), u.EvmStatePoller().LatestBlock(), + "per-upstream poller must advance") + assert.Equal(t, int64(90677359), network.EvmHighestLatestBlockNumber(ctx), + "network tip must advance before any client would see the WS head") + assert.GreaterOrEqual(t, network.deliveredLatestBlock.Load(), int64(90677359), + "process-local high-water mark must cover the delivered WS tip") +} + +// Only a head from an upstream the selection policy keeps eligible lifts +// "latest": a fallback-tier (cordoned) or unknown source still feeds its own +// poller, but must not advertise a block no eligible upstream reports. +func TestNetworkHandle_SuggestLatestBlock_OnlyTipCandidatesLiftLatest(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, ups, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3e8", + }) + require.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx)) + var fallback *upstream.Upstream + for _, u := range ups { + if u.Id() == "fallback-1" { + fallback = u + } + } + require.NotNil(t, fallback) + + handle := &networkHandle{nw: network} + handle.SuggestLatestBlock("ws:fallback-1", 1010) + handle.SuggestLatestBlock("ws:unknown", 1020) + + assert.Equal(t, int64(1010), fallback.EvmStatePoller().LatestBlock(), + "the fallback's own poller still advances") + assert.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx), + "a cordoned or unknown source must not lift latest") + + handle.SuggestLatestBlock("ws:primary-1", 1001) + assert.Equal(t, int64(1001), network.EvmHighestLatestBlockNumber(ctx), + "an eligible upstream's head floors latest") +} + +// The delivered-head floor is network-wide: a use-upstream-scoped request is +// answered from its group's own head. +func TestDeliveredHeadFloor_SkipsSelectorScopedRequests(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _ := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "fast-1", chainID: 123, latestBlock: 1050, tags: []string{"family:fast"}}, + {id: "fast-2", chainID: 123, latestBlock: 1050, tags: []string{"family:fast"}}, + {id: "slow-1", chainID: 123, latestBlock: 1000, tags: []string{"family:slow"}}, + {id: "slow-2", chainID: 123, latestBlock: 1000, tags: []string{"family:slow"}}, + }, &common.EvmServedTipConfig{}) + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`)) + req.SetDirectives(&common.RequestDirectives{UseUpstream: "family:slow"}) + slowCtx := context.WithValue(ctx, common.RequestContextKey, req) + + network.NoteObservedLatestBlock(ctx, 1051) + assert.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(slowCtx)) + assert.Equal(t, int64(1051), network.EvmHighestLatestBlockNumber(ctx)) +} + +// Projects sharing a process (and so a shared-state registry) must not floor +// each other's "latest" for the same chain. +func TestDeliveredHeadFloor_IsScopedPerProject(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _ := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "a", chainID: 123, latestBlock: 1000}, + }, &common.EvmServedTipConfig{}) + sibling, err := NewNetwork(ctx, &log.Logger, "sibling", network.cfg, + network.rateLimitersRegistry, network.upstreamsRegistry, network.metricsTracker, nil) + require.NoError(t, err) + require.NotNil(t, sibling.latestBlockShared) + + network.NoteObservedLatestBlock(ctx, 1001) + assert.Equal(t, int64(1001), network.latestBlockShared.GetValue()) + assert.Equal(t, int64(0), sibling.latestBlockShared.GetValue()) +} diff --git a/erpc/projects.go b/erpc/projects.go index cbcd5a905..ac9795c25 100644 --- a/erpc/projects.go +++ b/erpc/projects.go @@ -127,6 +127,15 @@ func (p *PreparedProject) AuthenticateConsumer(ctx context.Context, req *common. return nil, nil } +// AcceptsConsumerPayload reports whether any consumer auth strategy accepts +// ap for some method (see auth.AuthRegistry.AcceptsPayload). +func (p *PreparedProject) AcceptsConsumerPayload(ctx context.Context, req *common.NormalizedRequest, ap *auth.AuthPayload) error { + if p.consumerAuthRegistry != nil { + return p.consumerAuthRegistry.AcceptsPayload(ctx, req, ap) + } + return nil +} + func (p *PreparedProject) Forward(ctx context.Context, networkId string, nq *common.NormalizedRequest) (*common.NormalizedResponse, error) { start := time.Now() ctx, span := common.StartDetailSpan(ctx, "Project.Forward") diff --git a/erpc/subscription_manager.go b/erpc/subscription_manager.go new file mode 100644 index 000000000..be7e70c84 --- /dev/null +++ b/erpc/subscription_manager.go @@ -0,0 +1,459 @@ +package erpc + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/indexer" + "github.com/erpc/erpc/indexer/adapters/wsclient" + "github.com/erpc/erpc/indexer/adapters/wsupstream" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/upstream" + "github.com/rs/zerolog" +) + +const ( + MethodEthSubscribe = "eth_subscribe" + MethodEthUnsubscribe = "eth_unsubscribe" +) + +// unsubscribeTimeout bounds best-effort unsubscribes during connection +// cleanup. +const unsubscribeTimeout = 5 * time.Second + +// SubscriptionManager serves eth_subscribe and eth_unsubscribe on client +// WebSocket connections through the indexer. A network's upstream ingresses +// are registered the first time a client subscribes on it. +type SubscriptionManager struct { + logger *zerolog.Logger + idx *indexer.Indexer + + // connMu guards conns and every WsConnection.subsClosed, so no adapter + // is created for a connection after CleanupConnection. + connMu sync.Mutex + conns map[string]*connEntry // connId -> entry + + networks sync.Map // networkId -> struct{}, once bootstrapped + bootstrapMu sync.Mutex +} + +type connEntry struct { + adapter *wsclient.Adapter + detach func() +} + +func NewSubscriptionManager(logger *zerolog.Logger, idx *indexer.Indexer) *SubscriptionManager { + return &SubscriptionManager{ + logger: logger, + idx: idx, + conns: make(map[string]*connEntry), + } +} + +// IsSubscriptionMethod reports whether method is eth_subscribe or +// eth_unsubscribe. +func IsSubscriptionMethod(method string) bool { + return method == MethodEthSubscribe || method == MethodEthUnsubscribe +} + +func IsSubscribeMethod(method string) bool { + return method == MethodEthSubscribe +} + +// Subscribe handles eth_subscribe: it ensures the upstream subscription +// exists and registers a new client subscription on the connection. +func (sm *SubscriptionManager) Subscribe( + ctx context.Context, + wsc *WsConnection, + nq *common.NormalizedRequest, + project *PreparedProject, + networkId string, +) (*common.NormalizedResponse, error) { + start := time.Now() + method := MethodEthSubscribe + lg := sm.logger.With().Str("connId", wsc.id).Str("networkId", networkId).Logger() + + nw, err := project.GetNetwork(ctx, networkId) + if err != nil { + return nil, err + } + nq.SetNetwork(nw) + + if err := sm.bootstrapNetwork(ctx, nw); err != nil { + return nil, err + } + + conn := sm.getOrCreateConn(wsc) + if conn == nil { + return nil, wsclient.ErrClosed + } + // Fail fast before touching upstream filters; AddSubscription below + // enforces the limit atomically. + maxSubs := wsc.server.serverCfg.WebSocket.MaxSubscriptionsPerConnection + if conn.adapter.Count() >= maxSubs { + return nil, common.NewErrSubscriptionLimitExceeded(maxSubs) + } + + if err := sm.acquireRateLimits(ctx, project, nw, nq); err != nil { + return nil, err + } + + reqFinality := nq.Finality(ctx) + telemetry.CounterHandle(telemetry.MetricNetworkRequestsReceived, + project.Config.Id, nw.Label(), method, + reqFinality.String(), nq.UserId(), nq.AgentName(), + ).Inc() + + jrReq, err := nq.JsonRpcRequest() + if err != nil { + sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) + return nil, err + } + + subType := indexer.ExtractSubscriptionType(jrReq.Params) + clientSubID, err := indexer.GenerateClientSubID() + if err != nil { + err = fmt.Errorf("failed to generate subscription ID: %w", err) + sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) + return nil, err + } + + kind, filterHash, err := sm.resolveSubscription(ctx, networkId, subType, jrReq.Params) + if err != nil { + sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) + return nil, err + } + + if err := conn.adapter.AddSubscription(clientSubID, networkId, kind, filterHash, maxSubs); err != nil { + sm.releaseFilter(ctx, networkId, kind, filterHash) + if errors.Is(err, wsclient.ErrLimitExceeded) { + err = common.NewErrSubscriptionLimitExceeded(maxSubs) + } + sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) + return nil, err + } + + lg.Info(). + Str("clientSubId", clientSubID). + Str("subType", subType). + Msg("subscription established") + + sm.recordSuccessMetrics(project, nw, method, reqFinality, start, nq) + return newJsonRpcResultResponse(nq, jrReq, []byte(`"`+clientSubID+`"`)), nil +} + +// Unsubscribe handles an eth_unsubscribe request. +func (sm *SubscriptionManager) Unsubscribe( + ctx context.Context, + wsc *WsConnection, + nq *common.NormalizedRequest, + project *PreparedProject, + networkId string, +) (*common.NormalizedResponse, error) { + start := time.Now() + method := MethodEthUnsubscribe + lg := sm.logger.With().Str("connId", wsc.id).Str("networkId", networkId).Logger() + + nw, err := project.GetNetwork(ctx, networkId) + if err != nil { + return nil, err + } + nq.SetNetwork(nw) + + if err := sm.acquireRateLimits(ctx, project, nw, nq); err != nil { + return nil, err + } + + reqFinality := nq.Finality(ctx) + telemetry.CounterHandle(telemetry.MetricNetworkRequestsReceived, + project.Config.Id, nw.Label(), method, + reqFinality.String(), nq.UserId(), nq.AgentName(), + ).Inc() + + jrReq, err := nq.JsonRpcRequest() + if err != nil { + sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) + return nil, err + } + + clientSubID, err := indexer.ExtractClientSubID(jrReq.Params) + if err != nil { + sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) + return nil, err + } + + // Only the connection that created a subscription can remove it, and + // only the caller that actually removed it releases its filter. + sm.connMu.Lock() + conn := sm.conns[wsc.id] + sm.connMu.Unlock() + var kind indexer.EventKind + var subNetworkID, filterHash string + existed := false + if conn != nil { + kind, subNetworkID, filterHash, existed = conn.adapter.RemoveSubscription(clientSubID) + } + if !existed { + err := common.NewErrSubscriptionNotFound(clientSubID) + sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) + return nil, err + } + sm.releaseFilter(ctx, subNetworkID, kind, filterHash) + + lg.Info().Str("clientSubId", clientSubID).Str("subType", kind.String()).Msg("subscription removed") + + sm.recordSuccessMetrics(project, nw, method, reqFinality, start, nq) + return newJsonRpcResultResponse(nq, jrReq, []byte("true")), nil +} + +// CleanupConnection removes every subscription of a closed connection and +// detaches it from the indexer. Later Subscribe calls on it fail. +func (sm *SubscriptionManager) CleanupConnection(wsc *WsConnection) { + sm.connMu.Lock() + wsc.subsClosed = true + conn := sm.conns[wsc.id] + delete(sm.conns, wsc.id) + sm.connMu.Unlock() + if conn == nil { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), unsubscribeTimeout) + defer cancel() + for _, sub := range conn.adapter.Drain() { + sm.releaseFilter(ctx, sub.NetworkID, sub.Kind, sub.FilterHash) + } + conn.detach() + + sm.logger.Debug().Str("connId", wsc.id).Msg("cleaned up all subscriptions for connection") +} + +// releaseFilter drops a subscription's filter reference; newHeads +// subscriptions hold none. +func (sm *SubscriptionManager) releaseFilter(ctx context.Context, networkID string, kind indexer.EventKind, filterHash string) { + if kind != indexer.KindNewHead && filterHash != "" { + sm.idx.ReleaseFilter(ctx, networkID, kind.String(), filterHash) + } +} + +// bootstrapNetwork registers the network with the indexer, with an ingress +// per WebSocket upstream, once. +func (sm *SubscriptionManager) bootstrapNetwork(ctx context.Context, nw *Network) error { + networkID := nw.networkId + if _, ok := sm.networks.Load(networkID); ok { + return nil + } + sm.bootstrapMu.Lock() + defer sm.bootstrapMu.Unlock() + if _, ok := sm.networks.Load(networkID); ok { + return nil + } + + wsUpstreams := nw.upstreamsRegistry.GetWsUpstreams(ctx, networkID) + if len(wsUpstreams) == 0 { + return common.NewErrNoWsUpstreamAvailable(networkID) + } + + sm.idx.RegisterNetwork(&networkHandle{nw: nw}) + sm.idx.RegisterNetworkSelector(networkID, &subIngressSelector{nw: nw}) + var adapterOpts wsupstream.Options + if cfg := nw.cfg; cfg != nil && cfg.Evm != nil && cfg.Evm.StripSubscribeFromBlockZero != nil { + adapterOpts.StripSubscribeFromBlockZero = *cfg.Evm.StripSubscribeFromBlockZero + } + for _, up := range wsUpstreams { + adapter := wsupstream.New(up, networkID, sm.logger, adapterOpts) + if adapter == nil { + continue + } + if err := sm.idx.AddIngress(ctx, networkID, adapter); err != nil { + sm.logger.Warn().Err(err).Str("upstreamId", up.Id()). + Msg("failed to register upstream ingress with indexer") + } + } + sm.networks.Store(networkID, struct{}{}) + return nil +} + +// subIngressSelector picks the ingresses of a filter subscribe the way the +// HTTP path picks upstreams: by score for eth_subscribe, skipping upstreams +// whose circuit breaker is open. Fallback-tier upstreams form the fallback +// tier only when the network's failover is enabled, as over HTTP. +type subIngressSelector struct { + nw *Network +} + +func (s *subIngressSelector) Select(_, _ string, _ []interface{}) (defaults, fallbacks []string) { + ups, err := s.nw.upstreamsRegistry.GetSortedUpstreams(context.Background(), s.nw.networkId, MethodEthSubscribe) + if err != nil { + return nil, nil + } + + failoverOn := s.nw.cfg != nil && s.nw.cfg.Failover.Enabled() + for _, u := range ups { + up, ok := u.(*upstream.Upstream) + if !ok { + continue + } + cfg := up.Config() + if cfg == nil || !upstream.IsWsEndpoint(cfg.Endpoint) || up.IsDown(MethodEthSubscribe) { + continue + } + name := "ws:" + up.Id() + if failoverOn && cfg.HasTag(common.TagTierFallback) { + fallbacks = append(fallbacks, name) + continue + } + defaults = append(defaults, name) + } + return defaults, fallbacks +} + +// resolveSubscription maps subType to its event kind and, for filter +// subscriptions, takes a filter reference. +func (sm *SubscriptionManager) resolveSubscription(ctx context.Context, networkID, subType string, params []interface{}) (indexer.EventKind, string, error) { + switch subType { + case indexer.SubTypeNewHeads: + return indexer.KindNewHead, "", nil + case indexer.SubTypeLogs, indexer.SubTypeNewPendingTransactions: + hash, err := sm.idx.EnsureFilter(ctx, networkID, subType, params) + if err != nil { + return 0, "", err + } + kind := indexer.KindLog + if subType == indexer.SubTypeNewPendingTransactions { + kind = indexer.KindPendingTx + } + return kind, hash, nil + default: + return 0, "", common.NewErrJsonRpcExceptionInternal(0, common.JsonRpcErrorInvalidArgument, + fmt.Sprintf("unsupported subscription type: %q", subType), nil, nil) + } +} + +// getOrCreateConn returns the connection's entry, attaching a new egress +// on first use, or nil once the connection was cleaned up. +func (sm *SubscriptionManager) getOrCreateConn(wsc *WsConnection) *connEntry { + sm.connMu.Lock() + defer sm.connMu.Unlock() + if wsc.subsClosed { + return nil + } + if existing, ok := sm.conns[wsc.id]; ok { + return existing + } + adapter := wsclient.New(wsc.id, wsc, sm.logger, wsc.server.serverCfg.WebSocket.SubscriptionBufferSize) + entry := &connEntry{adapter: adapter, detach: sm.idx.Attach(adapter)} + sm.conns[wsc.id] = entry + return entry +} + +func (sm *SubscriptionManager) acquireRateLimits( + ctx context.Context, + project *PreparedProject, + nw *Network, + nq *common.NormalizedRequest, +) error { + if err := project.AcquireRateLimitPermit(ctx, nq); err != nil { + return err + } + return nw.acquireRateLimitPermit(ctx, nq) +} + +func newJsonRpcResultResponse(nq *common.NormalizedRequest, jrReq *common.JsonRpcRequest, result []byte) *common.NormalizedResponse { + jrr := &common.JsonRpcResponse{} + _ = jrr.SetID(jrReq.ID) + jrr.SetResult(result) + return common.NewNormalizedResponse().WithRequest(nq).WithJsonRpcResponse(jrr) +} + +func (sm *SubscriptionManager) recordSuccessMetrics( + project *PreparedProject, + nw *Network, + method string, + finality common.DataFinalityState, + start time.Time, + nq *common.NormalizedRequest, +) { + telemetry.CounterHandle(telemetry.MetricNetworkSuccessfulRequests, + project.Config.Id, nw.Label(), "proxy", "proxy", + method, "1", finality.String(), "false", nq.UserId(), nq.AgentName(), + ).Inc() + telemetry.ObserverHandle(telemetry.MetricNetworkRequestDuration, + project.Config.Id, nw.Label(), "proxy", "proxy", + method, finality.String(), nq.UserId(), + ).Observe(time.Since(start).Seconds()) +} + +func (sm *SubscriptionManager) recordFailureMetrics( + project *PreparedProject, + nw *Network, + method string, + finality common.DataFinalityState, + start time.Time, + nq *common.NormalizedRequest, + err error, +) { + telemetry.CounterHandle(telemetry.MetricNetworkFailedRequests, + project.Config.Id, nw.Label(), method, + "0", // no upstream attempts for client-facing subscription failures + common.ErrorFingerprint(err), + string(common.ClassifySeverity(err)), + finality.String(), + nq.UserId(), + nq.AgentName(), + ).Inc() + telemetry.ObserverHandle(telemetry.MetricNetworkRequestDuration, + project.Config.Id, nw.Label(), "", "", + method, finality.String(), nq.UserId(), + ).Observe(time.Since(start).Seconds()) +} + +// networkHandle adapts *Network to indexer.NetworkHandle. +type networkHandle struct { + nw *Network +} + +func (h *networkHandle) Id() string { return h.nw.networkId } + +// SuggestLatestBlock passes a head from the ingress "ws:" to +// that upstream's state poller. Once the poller has accepted it (a major +// jump is verified asynchronously first), a head from a tip candidate also +// advances the network's delivered-head floor before clients see it; a +// fallback-tier or policy-excluded upstream cannot lift "latest". +func (h *networkHandle) SuggestLatestBlock(sourceId string, blockNumber int64) { + upstreamID, ok := strings.CutPrefix(sourceId, "ws:") + if !ok { + return + } + ctx := context.Background() + for _, u := range h.nw.upstreamsRegistry.GetNetworkUpstreams(ctx, h.nw.networkId) { + if u.Id() != upstreamID { + continue + } + poller := u.EvmStatePoller() + if poller == nil || poller.IsObjectNull() { + return + } + poller.SuggestLatestBlock(blockNumber) + if poller.LatestBlock() < blockNumber { + return + } + for _, c := range h.nw.tipCandidateUpstreams(ctx, "*") { + if c.Id() == upstreamID { + h.nw.NoteObservedLatestBlock(h.nw.appCtx, blockNumber) + break + } + } + return + } +} + +var ( + _ wsclient.NotificationWriter = (*WsConnection)(nil) + _ indexer.NetworkHandle = (*networkHandle)(nil) +) diff --git a/erpc/ws_server.go b/erpc/ws_server.go new file mode 100644 index 000000000..64f04135b --- /dev/null +++ b/erpc/ws_server.go @@ -0,0 +1,625 @@ +package erpc + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "runtime/debug" + "sync" + "sync/atomic" + "time" + + "github.com/erpc/erpc/auth" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/indexer/adapters/wsclient" + "github.com/erpc/erpc/telemetry" + "github.com/gorilla/websocket" + "github.com/rs/zerolog" +) + +var wsConnCounter int64 + +// WsConnection is a client WebSocket connection, bound to one network. +type WsConnection struct { + id string + conn *websocket.Conn + ctx context.Context // cancelled on teardown; parent of every request + cancel context.CancelFunc + logger *zerolog.Logger + + server *HttpServer + project *PreparedProject + networkId string + // architecture is the URL's, for error bodies built before the request + // is bound to its network (as on the HTTP path). + architecture common.NetworkArchitecture + httpReq *http.Request // the upgrade request, for auth and headers + + // Write synchronization (gorilla/websocket requires synchronized writes) + writeMu sync.Mutex + + // sem bounds concurrent requests; inflight lets teardown wait for them. + sem chan struct{} + inflight sync.WaitGroup + + // stop records how the connection should end and stops the read loop; + // teardown (run by the read loop's goroutine) carries it out. + stopOnce sync.Once + stopped chan struct{} + closeCode int + closeReason string + closeGrace time.Duration + done chan struct{} + + closed atomic.Bool // no more writes once set + peerClosed atomic.Bool // the peer's close frame was already answered + + // subsClosed is guarded by SubscriptionManager.connMu. + subsClosed bool + + lastDropLogAt atomic.Int64 +} + +// handleWebSocket upgrades an HTTP connection to WebSocket and runs the +// read/write loops for the lifetime of the connection. +func (s *HttpServer) handleWebSocket( + w http.ResponseWriter, + r *http.Request, + lg *zerolog.Logger, + project *PreparedProject, + architecture string, + chainId string, +) { + wsCfg := s.serverCfg.WebSocket + + upgrader := websocket.Upgrader{ + ReadBufferSize: wsCfg.ReadBufferSize, + WriteBufferSize: wsCfg.WriteBufferSize, + CheckOrigin: func(r *http.Request) bool { + return checkWsOrigin(r, project) + }, + } + + wsConn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + lg.Error().Err(err).Msg("websocket upgrade failed") + return + } + + // Not derived from the app context: shutdown ends connections via stop() + // so in-flight requests still get their response. + connCtx, connCancel := context.WithCancel(context.Background()) + wsc := &WsConnection{ + id: fmt.Sprintf("ws-%d", atomic.AddInt64(&wsConnCounter, 1)), + conn: wsConn, + ctx: connCtx, + cancel: connCancel, + logger: lg, + server: s, + project: project, + networkId: fmt.Sprintf("%s:%s", architecture, chainId), + architecture: common.NetworkArchitecture(architecture), + httpReq: r, + sem: make(chan struct{}, wsCfg.MaxConcurrentRequestsPerConnection), + stopped: make(chan struct{}), + done: make(chan struct{}), + } + + lg.Info().Str("connId", wsc.id).Str("remoteAddr", r.RemoteAddr).Msg("websocket connection established") + + // Store before checking draining so shutdownWebSockets either sees this + // connection or we see its flag. + s.activeWsConns.Store(wsc.id, wsc) + if s.draining.Load() { + wsc.stop(websocket.CloseGoingAway, "server shutting down", 0) + } + + wsConn.SetReadLimit(wsCfg.MaxMessageSize) + + // A peer that sends neither a pong nor a request within pongWait is + // dropped; the read loop arms the deadline before every read. + pingInterval := wsCfg.PingInterval.Duration() + pongWait := 2 * pingInterval + wsConn.SetPongHandler(func(string) error { + lg.Trace().Str("connId", wsc.id).Msg("websocket pong received") + return wsConn.SetReadDeadline(time.Now().Add(pongWait)) + }) + + // As gorilla's default handler (echo the close frame), plus logging the + // peer's close code and recording that it was answered. + wsConn.SetCloseHandler(func(code int, text string) error { + lg.Info(). + Str("connId", wsc.id). + Int("closeCode", code). + Str("closeReason", text). + Msg("websocket close frame received from peer") + message := websocket.FormatCloseMessage(code, "") + _ = wsConn.WriteControl(websocket.CloseMessage, message, time.Now().Add(wsWriteDeadline)) + wsc.peerClosed.Store(true) + return nil + }) + + go wsc.pingLoop(pingInterval) + + wsc.readLoop(pongWait) + wsc.teardown() +} + +// isWebSocketUpgradeRequest reports whether r is a WebSocket opening +// handshake (RFC 6455 section 4.1). Every upgrade-specific code path uses +// this one check, so a request that merely resembles an upgrade is treated +// as ordinary HTTP everywhere. +func isWebSocketUpgradeRequest(r *http.Request) bool { + return r.Method == http.MethodGet && websocket.IsWebSocketUpgrade(r) +} + +// checkWsOrigin checks the upgrade request's origin against the project's +// CORS allowed origins; without a CORS config every origin is allowed. +func checkWsOrigin(r *http.Request, project *PreparedProject) bool { + if project == nil || project.Config.CORS == nil { + return true + } + + origin := r.Header.Get("Origin") + if origin == "" { + return true + } + + for _, allowedOrigin := range project.Config.CORS.AllowedOrigins { + match, err := common.WildcardMatch(allowedOrigin, origin) + if err != nil { + continue + } + if match { + return true + } + } + return false +} + +func (wsc *WsConnection) readLoop(pongWait time.Duration) { + for { + // Backpressure: don't read the next request until a slot is free. + select { + case wsc.sem <- struct{}{}: + case <-wsc.stopped: + return + } + + _ = wsc.conn.SetReadDeadline(time.Now().Add(pongWait)) + select { + case <-wsc.stopped: + // stop() may have set its deadline before ours. + <-wsc.sem + return + default: + } + + _, message, err := wsc.conn.ReadMessage() + if err != nil { + <-wsc.sem + select { + case <-wsc.stopped: + return + default: + } + // The error type tells a clean close, a dropped connection and + // an expired read deadline apart. + ev := wsc.logger.Info().Str("connId", wsc.id).Err(err). + Str("errType", fmt.Sprintf("%T", err)) + if ce, ok := err.(*websocket.CloseError); ok { + ev = ev.Int("closeCode", ce.Code).Str("closeReason", ce.Text) + } + if websocket.IsUnexpectedCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + ev.Msg("websocket read ended: unexpected close") + } else { + ev.Msg("websocket read ended") + } + return + } + + wsc.inflight.Add(1) + go func() { + defer func() { + <-wsc.sem + wsc.inflight.Done() + }() + wsc.handleMessage(message) + }() + } +} + +func (wsc *WsConnection) handleMessage(raw []byte) { + defer func() { + if rec := recover(); rec != nil { + telemetry.MetricUnexpectedPanicTotal.WithLabelValues( + "ws-request-handler", + fmt.Sprintf("project:%s network:%s", wsc.project.Config.Id, wsc.networkId), + common.ErrorFingerprint(rec), + ).Inc() + wsc.logger.Error(). + Interface("panic", rec). + Str("stack", string(debug.Stack())). + Str("connId", wsc.id). + Msg("unexpected panic in websocket message handler") + } + }() + + startedAt := time.Now() + ctx, cancel := context.WithTimeoutCause(wsc.ctx, wsc.server.reqMaxTimeout, ErrHandlerTimeout) + defer cancel() + + raw = bytes.TrimLeft(raw, " \t\r\n") + if len(raw) > 0 && raw[0] == '[' { + wsc.handleBatch(ctx, raw, &startedAt) + return + } + + out, notification := wsc.handleRequest(ctx, raw, &startedAt, false) + if notification { + if r, ok := out.(*common.NormalizedResponse); ok { + go r.Release() + } + return + } + switch v := out.(type) { + case *common.NormalizedResponse: + wsc.writeNormalizedResponse(v) + default: + if err := wsc.writeJSON(v); err != nil { + wsc.logger.Debug().Err(err).Str("connId", wsc.id).Msg("failed to write error response") + } + } +} + +// handleRequest runs one JSON-RPC request through the same steps as the +// HTTP handler and returns the response or error body to write. +// Subscription methods are rejected in batches. A panic becomes an error +// response carrying the request's id, since WebSocket clients match replies +// by id. +func (wsc *WsConnection) handleRequest(ctx context.Context, raw []byte, startedAt *time.Time, inBatch bool) (out interface{}, notification bool) { + defer func() { + if rec := recover(); rec != nil { + telemetry.MetricUnexpectedPanicTotal.WithLabelValues( + "ws-request-handler", + fmt.Sprintf("project:%s network:%s", wsc.project.Config.Id, wsc.networkId), + common.ErrorFingerprint(rec), + ).Inc() + wsc.logger.Error(). + Interface("panic", rec). + Str("stack", string(debug.Stack())). + Str("connId", wsc.id). + Msg("unexpected panic in websocket request handler") + // A fresh request: the panic may have left nq locked. + out = processErrorBody(wsc.logger, startedAt, common.NewNormalizedRequest(raw), + fmt.Errorf("unexpected server panic: %v", rec), wsc.server.serverCfg.IncludeErrorDetails, wsc.architecture) + } + }() + + nq := common.NewNormalizedRequest(raw) + nq.ForwardHeaders = make(http.Header) + requestCtx := common.StartRequestSpan(ctx, nq) + nq.SetClientIP(wsc.server.resolveRealClientIP(wsc.httpReq)) + + fail := func(err error, includeDetails *bool) interface{} { + common.EndRequestSpan(requestCtx, nil, err) + return processErrorBody(wsc.logger, startedAt, nq, err, includeDetails, wsc.architecture) + } + unsupported := func(message string) interface{} { + common.EndRequestSpan(requestCtx, nil, nil) + return wsc.buildUnsupportedMethodResponse(nq, message) + } + + if err := nq.Validate(); err != nil { + return fail(err, &common.TRUE), notification + } + // A valid request without an id is a notification: it runs, but gets no + // reply (JSON-RPC 2.0 §4.1). Invalid requests are still answered. + if jrr, err := nq.JsonRpcRequest(); err == nil { + notification = jrr.IsNotification() + } + + project := wsc.project + headers := wsc.httpReq.Header + queryArgs := wsc.httpReq.URL.Query() + if err := applyForwardHeaders(project, nq, headers); err != nil { + return fail(err, &common.TRUE), notification + } + + method, _ := nq.Method() + allowed, err := isMethodAllowed(project, method) + if err != nil { + return fail(err, &common.TRUE), notification + } + if !allowed { + return unsupported(fmt.Sprintf("method not supported: %s", method)), notification + } + if inBatch && IsSubscriptionMethod(method) { + return unsupported("subscription methods (eth_subscribe, eth_unsubscribe) are not supported in batch requests"), notification + } + + ap, err := auth.NewPayloadFromHttp(method, wsc.httpReq.RemoteAddr, headers, queryArgs) + if err != nil { + return fail(err, &common.TRUE), notification + } + user, err := project.AuthenticateConsumer(requestCtx, nq, method, ap) + if err != nil { + return fail(err, wsc.server.serverCfg.IncludeErrorDetails), notification + } + nq.SetUser(user) + if project.Config.TrustUserIdHeader && nq.User() == nil { + nq.SetUserFromTrustedHeader(headers.Get(common.HeaderUserId)) + } + + nw, err := project.GetNetwork(requestCtx, wsc.networkId) + if err != nil { + return fail(err, wsc.server.serverCfg.IncludeErrorDetails), notification + } + nq.SetNetwork(nw) + applyRequestDirectives(project, nw, nq, headers, queryArgs) + + var resp *common.NormalizedResponse + switch { + case IsSubscribeMethod(method): + resp, err = wsc.server.subscriptionManager.Subscribe(requestCtx, wsc, nq, project, wsc.networkId) + case IsSubscriptionMethod(method): + resp, err = wsc.server.subscriptionManager.Unsubscribe(requestCtx, wsc, nq, project, wsc.networkId) + default: + resp, err = project.Forward(requestCtx, wsc.networkId, nq) + } + if err != nil { + if resp != nil { + go resp.Release() + } + return fail(err, wsc.server.serverCfg.IncludeErrorDetails), notification + } + + common.EndRequestSpan(requestCtx, resp, nil) + return resp, notification +} + +func (wsc *WsConnection) handleBatch(ctx context.Context, raw []byte, startedAt *time.Time) { + var requests []json.RawMessage + if err := common.SonicCfg.Unmarshal(raw, &requests); err != nil { + wsc.writeError(int(common.JsonRpcErrorParseException), "parse error") + return + } + // JSON-RPC 2.0 section 6: an empty batch gets a single Invalid Request error. + if len(requests) == 0 { + wsc.writeError(int(common.JsonRpcErrorClientSideException), "invalid request: empty batch") + return + } + + responses := make([]interface{}, len(requests)) + notifications := make([]bool, len(requests)) + var wg sync.WaitGroup + + for i, reqBody := range requests { + wg.Add(1) + go func(index int, reqRaw json.RawMessage) { + defer wg.Done() + responses[index], notifications[index] = wsc.handleRequest(ctx, reqRaw, startedAt, true) + }(i, reqBody) + } + + wg.Wait() + + // Notifications get no entry; a batch of only notifications gets no reply. + replies := make([]interface{}, 0, len(responses)) + for i, resp := range responses { + if !notifications[i] { + replies = append(replies, resp) + } + } + if len(replies) > 0 { + wsc.writeBatchResponse(replies) + } + + for _, resp := range responses { + if r, ok := resp.(*common.NormalizedResponse); ok { + go r.Release() + } + } +} + +func (wsc *WsConnection) buildUnsupportedMethodResponse(nq *common.NormalizedRequest, message string) *HttpJsonRpcErrorResponse { + jsonrpcVersion := "2.0" + var reqId interface{} + if jrr, err := nq.JsonRpcRequest(); err == nil { + jsonrpcVersion = jrr.JSONRPC + reqId = jrr.ID + } + return &HttpJsonRpcErrorResponse{ + Jsonrpc: jsonrpcVersion, + Id: reqId, + Error: map[string]interface{}{ + "code": int(common.JsonRpcErrorUnsupportedException), + "message": message, + }, + } +} + +// wsWriteDeadline bounds every write, so a stalled client cannot hold +// writeMu and starve the connection's other writers. +const wsWriteDeadline = 10 * time.Second + +var errWsConnClosed = errors.New("connection closed") + +// writeMessage writes one text message produced by write. +func (wsc *WsConnection) writeMessage(write func(w io.Writer) error) error { + wsc.writeMu.Lock() + defer wsc.writeMu.Unlock() + + if wsc.closed.Load() { + return errWsConnClosed + } + if err := wsc.conn.SetWriteDeadline(time.Now().Add(wsWriteDeadline)); err != nil { + return err + } + defer wsc.conn.SetWriteDeadline(time.Time{}) + + w, err := wsc.conn.NextWriter(websocket.TextMessage) + if err != nil { + return err + } + err = write(w) + if closeErr := w.Close(); err == nil { + err = closeErr + } + return err +} + +func (wsc *WsConnection) writeJSON(v interface{}) error { + return wsc.writeMessage(func(w io.Writer) error { + return json.NewEncoder(w).Encode(v) + }) +} + +func (wsc *WsConnection) writeError(code int, message string) { + _ = wsc.writeJSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": nil, + "error": map[string]interface{}{ + "code": code, + "message": message, + }, + }) +} + +func (wsc *WsConnection) writeNormalizedResponse(resp *common.NormalizedResponse) { + // Release even when the write never starts (connection closed or broken). + defer func() { go resp.Release() }() + err := wsc.writeMessage(func(w io.Writer) error { + _, err := resp.WriteTo(w) + return err + }) + if err != nil { + wsc.logger.Debug().Err(err).Str("connId", wsc.id).Msg("failed to write websocket response") + } +} + +func (wsc *WsConnection) writeBatchResponse(responses []interface{}) { + _ = wsc.writeMessage(func(w io.Writer) error { + _, err := NewBatchResponseWriter(responses).WriteTo(w) + return err + }) +} + +func (wsc *WsConnection) pingLoop(interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // WriteControl is safe alongside other writers, so pings don't + // wait behind a slow data write holding writeMu. + err := wsc.conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(wsWriteDeadline)) + wsc.logger.Trace().Str("connId", wsc.id).Msg("websocket ping sent") + if err != nil { + wsc.logger.Info().Err(err).Str("connId", wsc.id). + Str("errType", fmt.Sprintf("%T", err)). + Msg("websocket ping failed, closing connection") + wsc.stop(websocket.CloseGoingAway, "", 0) + return + } + case <-wsc.stopped: + return + } + } +} + +// stop ends the connection with the given close code: it stops reading new +// requests and lets teardown finish. Requests already in flight get up to +// grace to complete. Only the first call counts; safe from any goroutine. +func (wsc *WsConnection) stop(code int, reason string, grace time.Duration) { + wsc.stopOnce.Do(func() { + wsc.closeCode, wsc.closeReason, wsc.closeGrace = code, reason, grace + close(wsc.stopped) + _ = wsc.conn.SetReadDeadline(time.Now()) + }) +} + +// teardown runs on the read loop's goroutine once it exits, so no request +// can start while it waits for in-flight ones. +func (wsc *WsConnection) teardown() { + // Server-initiated unless stop() said otherwise; never 1000, which + // would tell the client nothing went wrong. + wsc.stop(websocket.CloseGoingAway, "", 0) + + if wsc.closeGrace > 0 { + wsc.waitInflight(wsc.closeGrace) + } + // Cancelled requests still write their JSON-RPC error before the close. + wsc.cancel() + wsc.waitInflight(wsWriteDeadline) + wsc.closed.Store(true) + + wsc.server.subscriptionManager.CleanupConnection(wsc) + + if !wsc.peerClosed.Load() { + _ = wsc.conn.WriteControl( + websocket.CloseMessage, + websocket.FormatCloseMessage(wsc.closeCode, wsc.closeReason), + time.Now().Add(unsubscribeTimeout), + ) + } + _ = wsc.conn.Close() + wsc.server.activeWsConns.Delete(wsc.id) + close(wsc.done) + + wsc.logger.Info().Str("connId", wsc.id).Int("closeCode", wsc.closeCode).Msg("websocket connection closed") +} + +func (wsc *WsConnection) waitInflight(timeout time.Duration) { + idle := make(chan struct{}) + go func() { + wsc.inflight.Wait() + close(idle) + }() + select { + case <-idle: + case <-time.After(timeout): + } +} + +func (wsc *WsConnection) WriteSubscriptionNotification(clientSubId string, result json.RawMessage) error { + notification := map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": map[string]interface{}{ + "subscription": clientSubId, + "result": result, + }, + } + return wsc.writeJSON(notification) +} + +// NotificationDropped records a notification lost to a full subscription +// buffer. Losing one that is not lossy closes the connection with 1013 so +// the client reconnects knowing it missed data instead of silently +// diverging. +func (wsc *WsConnection) NotificationDropped(sub wsclient.Subscription, lossy bool) { + network := sub.NetworkID + if nw, err := wsc.project.GetNetwork(wsc.ctx, sub.NetworkID); err == nil { + network = nw.Label() + } + telemetry.CounterHandle(telemetry.MetricWebsocketSubscriptionNotificationsDroppedTotal, + wsc.project.Config.Id, network, sub.Kind.String(), + ).Inc() + + now := time.Now().UnixNano() + if last := wsc.lastDropLogAt.Load(); now-last > int64(10*time.Second) && wsc.lastDropLogAt.CompareAndSwap(last, now) { + wsc.logger.Warn().Str("connId", wsc.id).Str("clientSubId", sub.ClientSubID). + Str("kind", sub.Kind.String()).Bool("closingConnection", !lossy). + Msg("client is not keeping up; subscription notifications dropped") + } + + if !lossy { + wsc.stop(websocket.CloseTryAgainLater, "subscription buffer overflow: notifications were dropped", 0) + } +} diff --git a/erpc/ws_server_hardening_test.go b/erpc/ws_server_hardening_test.go new file mode 100644 index 000000000..86b9e9192 --- /dev/null +++ b/erpc/ws_server_hardening_test.go @@ -0,0 +1,538 @@ +package erpc + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/h2non/gock" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// +// --- Upgrade detection --- +// + +// A client advertising gzip (every browser, most WS libraries) must still be +// able to upgrade under the default config, where response gzip is enabled. +func TestWebSocket_UpgradeWithAcceptEncodingGzip(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := httpOnlyConfig() + cfg.Server.EnableGzip = util.BoolPtr(true) + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + hdr := http.Header{} + hdr.Set("Accept-Encoding", "gzip, deflate, br") + conn, resp, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123", addr), hdr) + require.NoError(t, err, "upgrade must succeed when the client accepts gzip") + defer conn.Close() + assert.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode) + + res := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", res["result"]) +} + +// A plain JSON-RPC POST that merely carries an Upgrade header is not a +// WebSocket handshake and must stay bounded by server.maxTimeout. +func TestWebSocket_UpgradeHeaderDoesNotBypassMaxTimeout(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + gock.New("http://rpc1.localhost"). + Post("/"). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getBalance") + }). + Reply(200). + Delay(3 * time.Second). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 1, "result": "0x1"}) + + cfg := httpOnlyConfig() + d := common.Duration(300 * time.Millisecond) + cfg.Server.MaxTimeout = &d + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://%s/test_ws/evm/123", addr), + strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Upgrade", "websocket") + + start := time.Now() + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + elapsed := time.Since(start) + + assert.Less(t, elapsed, 2*time.Second, "request must be cut off near maxTimeout, took %s (body=%s)", elapsed, body) + assert.Contains(t, string(body), "timeout") +} + +// Header tokens are case-insensitive (RFC 6455 section 4.2.1); a handshake +// spelled "Upgrade: WebSocket" must upgrade, not be answered as a +// healthcheck. +func TestWebSocket_UpgradeHeaderIsCaseInsensitive(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + raw, err := net.Dial("tcp", addr) + require.NoError(t, err) + defer raw.Close() + _ = raw.SetDeadline(time.Now().Add(5 * time.Second)) + + _, err = fmt.Fprintf(raw, "GET /test_ws/evm/123 HTTP/1.1\r\n"+ + "Host: %s\r\n"+ + "Upgrade: WebSocket\r\n"+ + "Connection: Upgrade\r\n"+ + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"+ + "Sec-WebSocket-Version: 13\r\n\r\n", addr) + require.NoError(t, err) + + resp, err := http.ReadResponse(bufio.NewReader(raw), nil) + require.NoError(t, err) + assert.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode) +} + +// +// --- Connection lifecycle --- +// + +// subscribingWsUpstream answers eth_subscribe/eth_unsubscribe on top of the +// state-poller methods. +func subscribingWsUpstream(t *testing.T) string { + t.Helper() + srv := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + result := interface{}("0x1") + switch method { + case "eth_subscribe": + result = "0xupstreamsub" + case "eth_unsubscribe": + result = true + } + _ = mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": result}) + }) + }) + t.Cleanup(srv.Close) + return "ws" + strings.TrimPrefix(srv.URL, "http") +} + +func wsConnCount(s *HttpServer) (active, subscribed int) { + s.activeWsConns.Range(func(_, _ interface{}) bool { active++; return true }) + s.subscriptionManager.connMu.Lock() + subscribed = len(s.subscriptionManager.conns) + s.subscriptionManager.connMu.Unlock() + return active, subscribed +} + +// A client that subscribes and immediately closes must not leave an egress +// adapter attached to the indexer (it would receive every event forever). +func TestWebSocket_SubscribeThenCloseDoesNotLeak(t *testing.T) { + setupGock() + defer util.ResetGock() + + srv, addr, cancel := startTestERPCServer(t, standardWsConfig(subscribingWsUpstream(t))) + defer func() { _ = srv.Shutdown(&log.Logger); cancel() }() + + for i := 0; i < 20; i++ { + conn := dialWs(t, addr) + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`))) + require.NoError(t, conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))) + _ = conn.Close() + } + + require.Eventually(t, func() bool { + active, subscribed := wsConnCount(srv) + return active == 0 && subscribed == 0 + }, 5*time.Second, 50*time.Millisecond, "connections must be fully cleaned up") +} + +// eth_unsubscribe only reaches the caller's own subscriptions. +func TestWebSocket_UnsubscribeIsScopedToConnection(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, standardWsConfig(subscribingWsUpstream(t))) + defer cleanup() + + connA := dialWs(t, addr) + defer connA.Close() + connB := dialWs(t, addr) + defer connB.Close() + + resp := sendAndReceive(t, connA, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + subID, ok := resp["result"].(string) + require.True(t, ok, "subscribe failed: %v", resp) + + unsub := fmt.Sprintf(`{"jsonrpc":"2.0","id":2,"method":"eth_unsubscribe","params":["%s"]}`, subID) + resp = sendAndReceive(t, connB, unsub) + assert.NotNil(t, resp["error"], "another connection must not remove the subscription") + + resp = sendAndReceive(t, connA, unsub) + assert.Equal(t, true, resp["result"], "owner must still be able to unsubscribe: %v", resp) +} + +// Shutdown closes client connections with 1001 as soon as it starts, not +// after waitBeforeShutdown, and lets in-flight requests answer first. +func TestWebSocket_ShutdownClosesPromptlyAfterInflightRequests(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + gock.New("http://rpc1.localhost"). + Post("/"). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getBalance") + }). + Reply(200). + Delay(500 * time.Millisecond). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 1, "result": "0xabc123"}) + + cfg := standardWsConfig(subscribingWsUpstream(t)) + cfg.Server.WaitBeforeShutdown = common.Duration(5 * time.Second).Ptr() + srv, addr, cancel := startTestERPCServer(t, cfg) + defer func() { cancel(); _ = srv.Shutdown(&log.Logger) }() + + subscriber := dialWs(t, addr) + defer subscriber.Close() + resp := sendAndReceive(t, subscriber, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp["result"], "subscribe failed: %v", resp) + + requester := dialWs(t, addr) + defer requester.Close() + require.NoError(t, requester.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","id":7,"method":"eth_getBalance","params":["0xaaaa","latest"]}`))) + time.Sleep(100 * time.Millisecond) // let the request start + + start := time.Now() + cancel() + + var closeErr *websocket.CloseError + _ = subscriber.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, _, err := subscriber.ReadMessage() + require.ErrorAs(t, err, &closeErr) + assert.Equal(t, websocket.CloseGoingAway, closeErr.Code) + assert.Less(t, time.Since(start), time.Second, "close must not wait for waitBeforeShutdown") + + _ = requester.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, msg, err := requester.ReadMessage() + require.NoError(t, err, "in-flight request must be answered before the close") + assert.Contains(t, string(msg), "0xabc123") + _, _, err = requester.ReadMessage() + require.ErrorAs(t, err, &closeErr) + assert.Equal(t, websocket.CloseGoingAway, closeErr.Code) + + // A draining server closes new connections straight away. + late := dialWs(t, addr) + defer late.Close() + _ = late.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, _, err = late.ReadMessage() + require.ErrorAs(t, err, &closeErr) + assert.Equal(t, websocket.CloseGoingAway, closeErr.Code) +} + +// A peer that never answers pings is disconnected. +func TestWebSocket_UnresponsivePeerIsDisconnected(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := httpOnlyConfig() + cfg.Server.WebSocket = &common.WebSocketServerConfig{PingInterval: common.Duration(200 * time.Millisecond).Ptr()} + srv, addr, cancel := startTestERPCServer(t, cfg) + defer func() { _ = srv.Shutdown(&log.Logger); cancel() }() + + conn := dialWs(t, addr) + defer conn.Close() + conn.SetPingHandler(func(string) error { return nil }) // never pong + go func() { + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } + }() + + require.Eventually(t, func() bool { + active, _ := wsConnCount(srv) + return active == 0 + }, 3*time.Second, 50*time.Millisecond) +} + +// A connection runs at most maxConcurrentRequestsPerConnection requests at +// once; the rest wait instead of failing. +func TestWebSocket_ConcurrentRequestsAreBounded(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + gock.New("http://rpc1.localhost"). + Post("/"). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_getBalance") + }). + Reply(200). + Delay(300 * time.Millisecond). + JSON(map[string]interface{}{"jsonrpc": "2.0", "id": 1, "result": "0xabc123"}) + + cfg := httpOnlyConfig() + cfg.Server.WebSocket = &common.WebSocketServerConfig{MaxConcurrentRequestsPerConnection: 1} + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + start := time.Now() + for i := 0; i < 3; i++ { + // Distinct params so multiplexing can't merge them. + msg := fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"eth_getBalance","params":["0x%d","latest"]}`, i, i) + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(msg))) + } + for i := 0; i < 3; i++ { + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err) + assert.Contains(t, string(msg), "0xabc123") + } + assert.GreaterOrEqual(t, time.Since(start), 900*time.Millisecond, "requests must run one at a time") +} + +// +// --- Error shapes --- +// + +func TestWebSocket_ErrorShapes(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, standardWsConfig(subscribingWsUpstream(t))) + defer cleanup() + conn := dialWs(t, addr) + defer conn.Close() + + t.Run("UnknownSubscriptionTypeIsInvalidParams", func(t *testing.T) { + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newBananas"]}`) + require.NotNil(t, resp["error"]) + assert.EqualValues(t, common.JsonRpcErrorInvalidArgument, resp["error"].(map[string]interface{})["code"]) + }) + + t.Run("BatchWithLeadingWhitespace", func(t *testing.T) { + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(" \n[{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBalance\",\"params\":[\"0xaaaa\",\"latest\"]}]"))) + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err) + assert.True(t, strings.HasPrefix(string(msg), "["), "expected a batch response, got %s", msg) + assert.Contains(t, string(msg), `"result"`) + }) + + t.Run("EmptyBatchIsSingleInvalidRequest", func(t *testing.T) { + resp := sendAndReceive(t, conn, `[]`) + require.NotNil(t, resp["error"]) + assert.EqualValues(t, common.JsonRpcErrorClientSideException, resp["error"].(map[string]interface{})["code"]) + }) +} + +// A WebSocket connection is bound to one network, so a path that doesn't +// name one is refused at the handshake. +func TestWebSocket_UpgradeRequiresNetwork(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + _, resp, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws", addr), nil) + require.Error(t, err) + require.NotNil(t, resp) + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + +// A request that panics must still get a reply with its own id: WebSocket +// clients match replies by id, so without one they wait for their timeout. +func TestWebSocket_PanickedRequestRepliesWithItsId(t *testing.T) { + // No upgrade request: resolving the client IP panics inside handleRequest. + wsc := &WsConnection{ + id: "test", + logger: &log.Logger, + networkId: "evm:123", + server: &HttpServer{serverCfg: &common.ServerConfig{}}, + project: &PreparedProject{Config: &common.ProjectConfig{Id: "main"}}, + } + startedAt := time.Now() + + out, _ := wsc.handleRequest(t.Context(), []byte(`{"jsonrpc":"2.0","id":7,"method":"eth_chainId"}`), &startedAt, false) + require.NotNil(t, out) + body, err := common.SonicCfg.Marshal(out) + require.NoError(t, err) + assert.Contains(t, string(body), `"id":7`) + assert.Contains(t, string(body), `"error"`) +} + +// Every message on a WebSocket connection authenticates with the upgrade +// request's credentials, so an upgrade whose credentials no strategy accepts +// is refused instead of holding a connection that can never be served. +// Method scoping still applies per message. +func TestWebSocket_UpgradeRequiresAcceptedCredentials(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := httpOnlyConfig() + cfg.Projects[0].Auth = &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{{ + Type: common.AuthTypeSecret, + Secret: &common.SecretStrategyConfig{Value: "s3cret"}, + IgnoreMethods: []string{"*"}, + AllowMethods: []string{"eth_getBalance"}, + }}} + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + _, resp, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123", addr), nil) + require.Error(t, err, "an upgrade without credentials must be refused") + require.NotNil(t, resp) + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) + + _, resp, err = websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123?secret=wrong", addr), nil) + require.Error(t, err, "an upgrade with a rejected secret must be refused") + require.NotNil(t, resp) + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) + + conn, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/evm/123?secret=s3cret", addr), nil) + require.NoError(t, err) + defer conn.Close() + + ok := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x0000000000000000000000000000000000000001","latest"]}`) + assert.Equal(t, "0xabc123", ok["result"]) + + // The strategy only covers eth_getBalance; other methods stay unauthorized. + denied := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":2,"method":"eth_blockNumber","params":[]}`) + assert.NotNil(t, denied["error"], "method scoping must still apply per message") + assert.Nil(t, denied["result"]) +} + +// closeRecorder is a response body that records whether it was closed. +type closeRecorder struct{ closed atomic.Bool } + +func (c *closeRecorder) Read([]byte) (int, error) { return 0, io.EOF } +func (c *closeRecorder) Close() error { c.closed.Store(true); return nil } + +// A response is released even when its write never starts, as on the batch +// and HTTP paths. +func TestWebSocket_ResponseReleasedWhenWriteNeverStarts(t *testing.T) { + wsc := &WsConnection{ + id: "test", + logger: &log.Logger, + server: &HttpServer{serverCfg: &common.ServerConfig{}}, + } + wsc.closed.Store(true) + + body := &closeRecorder{} + wsc.writeNormalizedResponse(common.NewNormalizedResponse().WithBody(body)) + assert.Eventually(t, body.closed.Load, time.Second, 10*time.Millisecond) +} + +// Errors raised before a WebSocket request is bound to its network carry +// the connection's architecture, as on HTTP: an SVM consumer rate-limit +// error uses the SVM wire code, not -32005 (NodeUnhealthy to Solana clients). +func TestWebSocket_SvmAuthRateLimitUsesSvmWireCode(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + + cfg := &common.Config{ + Server: &common.ServerConfig{ListenV4: util.BoolPtr(true)}, + Projects: []*common.ProjectConfig{{ + Id: "test_ws", + Networks: []*common.NetworkConfig{{ + Architecture: common.ArchitectureSvm, + Svm: &common.SvmNetworkConfig{Cluster: "mainnet-beta"}, + }}, + Upstreams: []*common.UpstreamConfig{{ + Id: "svm-ws-test", + Type: common.UpstreamTypeSvm, + Endpoint: "http://svm-ws-rpc1.localhost", + Svm: &common.SvmUpstreamConfig{Cluster: "mainnet-beta"}, + }}, + Auth: &common.AuthConfig{Strategies: []*common.AuthStrategyConfig{{ + Type: common.AuthTypeSecret, + Secret: &common.SecretStrategyConfig{Value: "s3cret"}, + RateLimitBudget: "one-per-minute", + }}}, + }}, + RateLimiters: &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{{ + Id: "one-per-minute", + Rules: []*common.RateLimitRuleConfig{{Method: "*", MaxCount: 1, Period: common.RateLimitPeriodMinute}}, + }}}, + } + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + conn, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/test_ws/svm/mainnet-beta?secret=s3cret", addr), nil) + require.NoError(t, err) + defer conn.Close() + + // Both run concurrently: one spends the budget (and may wait on the + // unreachable upstream), the other is rejected at auth straight away. + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","id":1,"method":"getSlot","params":[]}`))) + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","id":2,"method":"getSlot","params":[]}`))) + var limited map[string]interface{} + require.NoError(t, conn.SetReadDeadline(time.Now().Add(10*time.Second))) + for limited == nil { + _, msg, err := conn.ReadMessage() + require.NoError(t, err) + var resp map[string]interface{} + require.NoError(t, json.Unmarshal(msg, &resp)) + if _, isErr := resp["error"]; isErr { + limited = resp + } + } + errObj, ok := limited["error"].(map[string]interface{}) + require.True(t, ok, "one request must be rate limited: %v", limited) + assert.EqualValues(t, -32000, errObj["code"]) +} + +// A request without an id is a notification: it runs but gets no reply +// (JSON-RPC 2.0 §4.1), alone or in a batch. "id": null is a request and is +// answered. +func TestWebSocket_NotificationsGetNoReply(t *testing.T) { + setupGock() + defer util.ResetGock() + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + conn := dialWs(t, addr) + defer conn.Close() + + const params = `"params":["0x0000000000000000000000000000000000000001","latest"]` + + nullID := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":null,"method":"eth_getBalance",`+params+`}`) + assert.Equal(t, "0xabc123", nullID["result"], `"id": null is a request and must be answered`) + + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(`{"jsonrpc":"2.0","method":"eth_getBalance",`+params+`}`))) + reply := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":7,"method":"eth_getBalance",`+params+`}`) + assert.EqualValues(t, 7, reply["id"], "the first reply must be the request's, not the notification's") + require.NoError(t, conn.WriteMessage(websocket.TextMessage, []byte(`[{"jsonrpc":"2.0","method":"eth_getBalance",`+params+`},{"jsonrpc":"2.0","method":"eth_getBalance",`+params+`}]`))) + + // Neither the notification nor the batch of notifications is answered. + // (A gorilla read timeout ends the connection's reads, so check once.) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(500*time.Millisecond))) + _, msg, err := conn.ReadMessage() + require.Error(t, err, "notifications must not be answered, got %s", msg) +} diff --git a/erpc/ws_server_selfheal_test.go b/erpc/ws_server_selfheal_test.go new file mode 100644 index 000000000..3e6218b7f --- /dev/null +++ b/erpc/ws_server_selfheal_test.go @@ -0,0 +1,234 @@ +package erpc + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// selfHealMockUpstream is a WS upstream whose connections can be killed +// WITHOUT a WebSocket close handshake, and which keeps accepting fresh +// connections afterwards — the moral equivalent of a node behind a +// still-healthy proxy being replaced. +type selfHealMockUpstream struct { + mu sync.Mutex + conns []*selfHealConn + connSeen chan *selfHealConn + subSeen chan string // upstream-assigned sub id per eth_subscribe served + subCount atomic.Int64 + lastSubID atomic.Value // string +} + +type selfHealConn struct { + conn *websocket.Conn + writeMu sync.Mutex +} + +func (sc *selfHealConn) writeJSON(v interface{}) { + sc.writeMu.Lock() + defer sc.writeMu.Unlock() + _ = sc.conn.SetWriteDeadline(time.Now().Add(2 * time.Second)) + _ = sc.conn.WriteJSON(v) +} + +func (sc *selfHealConn) sendHead(subID string, num int64) { + sc.writeJSON(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": map[string]interface{}{ + "subscription": subID, + "result": map[string]interface{}{ + "number": fmt.Sprintf("0x%x", num), + "hash": fmt.Sprintf("0x%064x", num), + "parentHash": fmt.Sprintf("0x%064x", num-1), + }, + }, + }) +} + +func (m *selfHealMockUpstream) handle(conn *websocket.Conn) { + sc := &selfHealConn{conn: conn} + m.mu.Lock() + m.conns = append(m.conns, sc) + m.mu.Unlock() + select { + case m.connSeen <- sc: + default: + } + + for { + _, msg, err := conn.ReadMessage() + if err != nil { + return + } + var req map[string]interface{} + if err := json.Unmarshal(msg, &req); err != nil { + continue + } + method, _ := req["method"].(string) + id := req["id"] + switch method { + case "eth_chainId": + sc.writeJSON(map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x7b"}) + case "eth_getBlockByNumber": + sc.writeJSON(map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": map[string]interface{}{"number": "0x100", "timestamp": "0x6702a8f0"}}) + case "eth_syncing": + sc.writeJSON(map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": false}) + case "eth_subscribe": + subID := fmt.Sprintf("0xupsub%02x", m.subCount.Add(1)) + m.lastSubID.Store(subID) + sc.writeJSON(map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": subID}) + select { + case m.subSeen <- subID: + default: + } + case "eth_unsubscribe": + sc.writeJSON(map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": true}) + default: + sc.writeJSON(map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + } +} + +// TestWebSocket_UpstreamDiesUngracefully_SelfHeals: when the only WS +// upstream's connection dies without a close handshake, eRPC must, with no +// restart: +// +// 1. detect the dead connection and re-dial, +// 2. re-subscribe newHeads upstream (retrying through transient failures), +// 3. resume delivering heads to the ALREADY-CONNECTED client subscription, +// 4. accept NEW client subscriptions afterwards. +func TestWebSocket_UpstreamDiesUngracefully_SelfHeals(t *testing.T) { + mock := &selfHealMockUpstream{ + connSeen: make(chan *selfHealConn, 16), + subSeen: make(chan string, 16), + } + mockSrv := mockWsUpstream(t, mock.handle) + defer mockSrv.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockSrv.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + defer cleanup() + + // eRPC's upstream WS client connects at upstream registration. + var upConn1 *selfHealConn + select { + case upConn1 = <-mock.connSeen: + case <-time.After(5 * time.Second): + t.Fatal("eRPC never connected to the WS upstream") + } + + // Client subscribes to newHeads (this lazily bootstraps the network's + // indexer ingress, which issues the upstream eth_subscribe). + client := dialWs(t, addr) + defer client.Close() + resp := sendAndReceive(t, client, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.Nil(t, resp["error"], "subscribe must succeed while the upstream is healthy: %v", resp["error"]) + clientSubID, ok := resp["result"].(string) + require.True(t, ok, "subscription ID should be a string, got: %v", resp["result"]) + + var sub1 string + select { + case sub1 = <-mock.subSeen: + case <-time.After(5 * time.Second): + t.Fatal("eRPC never subscribed newHeads on the upstream") + } + + // pushHeadsUntilReceived repeatedly emits fresh heads on the upstream + // connection until one reaches the given client (the upstream subscribe + // response and eRPC's handler registration race the first push — real + // chains emit heads continuously, so a retry loop models reality). + headNum := int64(0x200) + pushHeadsUntilReceived := func(c *websocket.Conn, up *selfHealConn, subID string, within time.Duration) map[string]interface{} { + t.Helper() + deadline := time.Now().Add(within) + notifCh := make(chan map[string]interface{}, 1) + errCh := make(chan error, 1) + go func() { + _ = c.SetReadDeadline(deadline) + _, msg, err := c.ReadMessage() + if err != nil { + errCh <- err + return + } + var notif map[string]interface{} + if err := json.Unmarshal(msg, ¬if); err != nil { + errCh <- err + return + } + notifCh <- notif + }() + for { + headNum++ + up.sendHead(subID, headNum) + select { + case notif := <-notifCh: + return notif + case err := <-errCh: + t.Fatalf("client read failed while waiting for a head: %v", err) + return nil + case <-time.After(250 * time.Millisecond): + if time.Now().After(deadline) { + t.Fatalf("client never received a head within %s", within) + return nil + } + } + } + } + + // Sanity: a head flows end-to-end before the failure. + notif := pushHeadsUntilReceived(client, upConn1, sub1, 5*time.Second) + require.Equal(t, "eth_subscription", notif["method"]) + require.Equal(t, clientSubID, notif["params"].(map[string]interface{})["subscription"]) + + // ── Failure injection ───────────────────────────────────────────── + // Kill the upstream TCP connection with NO WebSocket close handshake. + _ = upConn1.conn.UnderlyingConn().Close() + + // eRPC must re-dial (the httptest server is still up, as with a proxy + // in front of a replaced node) ... + var upConn2 *selfHealConn + select { + case upConn2 = <-mock.connSeen: + case <-time.After(10 * time.Second): + t.Fatal("eRPC never re-dialed the upstream after the ungraceful kill") + } + + // ... and re-subscribe newHeads with a fresh upstream subscription. + var sub2 string + select { + case sub2 = <-mock.subSeen: + case <-time.After(10 * time.Second): + t.Fatal("eRPC never re-subscribed newHeads after reconnecting") + } + assert.NotEqual(t, sub1, sub2, "the re-subscribe must be a fresh upstream subscription") + + // The ALREADY-CONNECTED client must resume receiving heads on its + // ORIGINAL subscription ID, with no client-side action. + notif = pushHeadsUntilReceived(client, upConn2, sub2, 10*time.Second) + assert.Equal(t, "eth_subscription", notif["method"]) + assert.Equal(t, clientSubID, notif["params"].(map[string]interface{})["subscription"], + "recovered heads must arrive on the client's original subscription ID") + + // And a NEW client must be able to subscribe and receive heads. + client2 := dialWs(t, addr) + defer client2.Close() + resp2 := sendAndReceive(t, client2, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.Nil(t, resp2["error"], "new client subscribe must succeed after recovery: %v", resp2["error"]) + client2SubID, _ := resp2["result"].(string) + + notif2 := pushHeadsUntilReceived(client2, upConn2, sub2, 10*time.Second) + assert.Equal(t, client2SubID, notif2["params"].(map[string]interface{})["subscription"]) +} diff --git a/erpc/ws_server_test.go b/erpc/ws_server_test.go new file mode 100644 index 000000000..919ac6cbb --- /dev/null +++ b/erpc/ws_server_test.go @@ -0,0 +1,1562 @@ +package erpc + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/util" + "github.com/gorilla/websocket" + "github.com/h2non/gock" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// +// --- Test helpers --- +// + +func init() { + util.ConfigureTestLogger() +} + +// mockWriteMus serialises writes per mock-upstream connection: gorilla +// supports one concurrent writer, and several mock handlers push +// notifications from a goroutine while the read loop answers requests. +var mockWriteMus sync.Map // *websocket.Conn -> *sync.Mutex + +// mockWriteJSON is the only way mock upstream handlers write to their +// connection. +func mockWriteJSON(conn *websocket.Conn, v interface{}) error { + muRaw, _ := mockWriteMus.LoadOrStore(conn, &sync.Mutex{}) + mu := muRaw.(*sync.Mutex) + mu.Lock() + defer mu.Unlock() + return conn.WriteJSON(v) +} + +// mockWsUpstream creates a test HTTP server that upgrades to WebSocket +// and delegates all message handling to the provided callback. +func mockWsUpstream(t *testing.T, handler func(conn *websocket.Conn)) *httptest.Server { + t.Helper() + upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Logf("mock ws upstream upgrade error: %v", err) + return + } + defer c.Close() + defer mockWriteMus.Delete(c) + handler(c) + })) + return srv +} + +// standardWsConfig returns a config with both an HTTP upstream (gock) and a WS upstream. +func standardWsConfig(wsURL string) *common.Config { + return &common.Config{ + Server: &common.ServerConfig{ + ListenV4: util.BoolPtr(true), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_ws", + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Id: "http-upstream", + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + { + Id: "ws-upstream", + Type: common.UpstreamTypeEvm, + Endpoint: wsURL, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } +} + +// multiWsConfig returns a config with one HTTP upstream (gock) and N WS upstreams. +func multiWsConfig(wsURLs ...string) *common.Config { + upstreams := []*common.UpstreamConfig{ + { + Id: "http-upstream", + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + } + for i, wsURL := range wsURLs { + upstreams = append(upstreams, &common.UpstreamConfig{ + Id: fmt.Sprintf("ws-upstream-%d", i), + Type: common.UpstreamTypeEvm, + Endpoint: wsURL, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }) + } + return &common.Config{ + Server: &common.ServerConfig{ListenV4: util.BoolPtr(true)}, + Projects: []*common.ProjectConfig{ + { + Id: "test_ws", + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: upstreams, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } +} + +// httpOnlyConfig returns a config with only HTTP upstreams (no WS). +func httpOnlyConfig() *common.Config { + return &common.Config{ + Server: &common.ServerConfig{ + ListenV4: util.BoolPtr(true), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_ws", + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } +} + +// setupGock sets up standard gock mocks (eth_getBalance) and EVM state poller stubs. +func setupGock() { + util.ResetGock() + util.SetupMocksForEvmStatePoller() + + gock.New("http://rpc1.localhost"). + Post("/"). + Persist(). + Filter(func(request *http.Request) bool { + body := util.SafeReadBody(request) + return strings.Contains(body, "eth_getBalance") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "result": "0xabc123", + }) +} + +// dialWs connects to the eRPC WebSocket endpoint for the test project. +func dialWs(t *testing.T, addr string) *websocket.Conn { + t.Helper() + wsURL := fmt.Sprintf("ws://%s/test_ws/evm/123", addr) + conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + require.NoError(t, err, "WebSocket dial should succeed") + return conn +} + +// sendAndReceive sends a JSON-RPC request string and reads the JSON response. +func sendAndReceive(t *testing.T, conn *websocket.Conn, req string) map[string]interface{} { + t.Helper() + err := conn.WriteMessage(websocket.TextMessage, []byte(req)) + require.NoError(t, err) + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err) + var resp map[string]interface{} + require.NoError(t, json.Unmarshal(msg, &resp)) + return resp +} + +// setupTestERPCServer boots an eRPC instance with the given config, returns +// the listen address and a cleanup function that shuts everything down. +func setupTestERPCServer(t *testing.T, cfg *common.Config) (string, context.CancelFunc) { + t.Helper() + httpServer, addr, cancel := startTestERPCServer(t, cfg) + cleanup := func() { + _ = httpServer.Shutdown(&log.Logger) + cancel() + } + return addr, cleanup +} + +// startTestERPCServer is setupTestERPCServer for tests that need the server +// itself or to cancel the app context on their own. +func startTestERPCServer(t *testing.T, cfg *common.Config) (*HttpServer, string, context.CancelFunc) { + t.Helper() + + logger := log.Logger + ctx, cancel := context.WithCancel(context.Background()) + + err := cfg.SetDefaults(&common.DefaultOptions{}) + require.NoError(t, err) + + ssr, err := data.NewSharedStateRegistry(ctx, &logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{ + MaxItems: 100_000, MaxTotalSize: "1GB", + }, + }, + }) + require.NoError(t, err) + + erpcInstance, err := NewERPC(ctx, &logger, ssr, nil, nil, cfg) + require.NoError(t, err) + + erpcInstance.Bootstrap(ctx) + + httpServer, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, cfg.Indexer, erpcInstance) + require.NoError(t, err) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := listener.Addr().(*net.TCPAddr).Port + + go func() { + err := httpServer.serverV4.Serve(listener) + if err != nil && err != http.ErrServerClosed { + t.Errorf("Server error: %v", err) + } + }() + + return httpServer, fmt.Sprintf("127.0.0.1:%d", port), cancel +} + +// standardMockWsHandler handles the common set of state poller methods +// (eth_chainId, eth_getBlockByNumber, eth_syncing) that the upstream +// must respond to before the WS client is considered ready. +func standardMockWsHandler(conn *websocket.Conn, customHandler func(method string, id interface{}, req map[string]interface{})) { + for { + _, msg, err := conn.ReadMessage() + if err != nil { + return + } + var req map[string]interface{} + if err := json.Unmarshal(msg, &req); err != nil { + continue + } + method, _ := req["method"].(string) + id := req["id"] + + switch method { + case "eth_chainId": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x7b"}) + case "eth_getBlockByNumber": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": map[string]interface{}{"number": "0x100", "timestamp": "0x6702a8f0"}}) + case "eth_syncing": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": false}) + default: + if customHandler != nil { + customHandler(method, id, req) + } else { + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + } + } +} + +// +// --- Tests: basic JSON-RPC over WebSocket --- +// + +func TestWebSocket_BasicRPC(t *testing.T) { + // Verifies a single JSON-RPC request/response over a WebSocket connection + t.Run("SingleRequestOverWebSocket", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0x1234567890abcdef1234567890abcdef12345678","latest"]}`) + assert.Equal(t, "2.0", resp["jsonrpc"]) + assert.Equal(t, float64(1), resp["id"]) + assert.Equal(t, "0xabc123", resp["result"]) + }) + + // Verifies sequential requests on the same connection work correctly + t.Run("MultipleRequestsOnSameConnection", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + gock.New("http://rpc1.localhost"). + Post("/"). + Persist(). + Filter(func(request *http.Request) bool { + body := util.SafeReadBody(request) + return strings.Contains(body, "eth_chainId") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 2, + "result": "0x7b", + }) + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + resp1 := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", resp1["result"]) + + resp2 := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":2,"method":"eth_chainId","params":[]}`) + assert.Equal(t, "0x7b", resp2["result"]) + }) + + // Verifies many concurrent writes/reads on a single connection + t.Run("ConcurrentRequestsOnSameConnection", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + const numRequests = 10 + var writeMu sync.Mutex + + for i := 0; i < numRequests; i++ { + writeMu.Lock() + msg := fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"eth_getBalance","params":["0xaaaa","latest"]}`, i) + err := conn.WriteMessage(websocket.TextMessage, []byte(msg)) + writeMu.Unlock() + require.NoError(t, err) + } + + responses := make(map[float64]bool) + for i := 0; i < numRequests; i++ { + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err) + + var resp map[string]interface{} + require.NoError(t, json.Unmarshal(msg, &resp)) + assert.Equal(t, "2.0", resp["jsonrpc"]) + if id, ok := resp["id"].(float64); ok { + responses[id] = true + } + } + + assert.Equal(t, numRequests, len(responses), "should receive all responses") + }) + + // Verifies JSON array batch requests work over WebSocket + t.Run("BatchRequestOverWebSocket", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + gock.New("http://rpc1.localhost"). + Post("/"). + Persist(). + Filter(func(request *http.Request) bool { + body := util.SafeReadBody(request) + return strings.Contains(body, "eth_chainId") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 2, + "result": "0x7b", + }) + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + batch := `[{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]},{"jsonrpc":"2.0","id":2,"method":"eth_chainId","params":[]}]` + err := conn.WriteMessage(websocket.TextMessage, []byte(batch)) + require.NoError(t, err) + + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, respMsg, err := conn.ReadMessage() + require.NoError(t, err) + + var responses []map[string]interface{} + err = json.Unmarshal(respMsg, &responses) + require.NoError(t, err, "Response should be a JSON array") + assert.Equal(t, 2, len(responses), "Batch response should contain 2 items") + }) + + // Verifies the server handles client disconnect and allows reconnection + t.Run("WebSocketClientDisconnect", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Contains(t, resp["result"], "0xabc123") + + err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + assert.NoError(t, err) + conn.Close() + + time.Sleep(200 * time.Millisecond) + conn2 := dialWs(t, addr) + defer conn2.Close() + + resp2 := sendAndReceive(t, conn2, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Contains(t, resp2["result"], "0xabc123") + }) + + // Verifies HTTP and WebSocket work simultaneously on the same server + t.Run("HTTPStillWorksAlongsideWebSocket", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + wsResp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", wsResp["result"]) + + httpURL := fmt.Sprintf("http://%s/test_ws/evm/123", addr) + cleanClient := &http.Client{Transport: &http.Transport{}} + httpResp, err := cleanClient.Post(httpURL, "application/json", strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`)) + require.NoError(t, err) + defer httpResp.Body.Close() + + assert.Equal(t, http.StatusOK, httpResp.StatusCode) + }) +} + +// +// --- Tests: error handling --- +// + +func TestWebSocket_ErrorHandling(t *testing.T) { + // Verifies invalid JSON returns a parse error response + t.Run("InvalidJSON", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + err := conn.WriteMessage(websocket.TextMessage, []byte(`{not valid json`)) + require.NoError(t, err) + + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err) + + var resp map[string]interface{} + require.NoError(t, json.Unmarshal(msg, &resp)) + assert.NotNil(t, resp["error"], "should return error for invalid JSON") + }) + + // Verifies an empty message body returns an error response + t.Run("EmptyBody", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + err := conn.WriteMessage(websocket.TextMessage, []byte(``)) + require.NoError(t, err) + + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err) + + var resp map[string]interface{} + require.NoError(t, json.Unmarshal(msg, &resp)) + assert.NotNil(t, resp["error"], "should return error for empty body") + }) + + // Verifies upstream JSON-RPC errors are forwarded to the client + t.Run("UpstreamError", func(t *testing.T) { + util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.ResetGock() + + gock.New("http://rpc1.localhost"). + Post("/"). + Persist(). + Filter(func(request *http.Request) bool { + body := util.SafeReadBody(request) + return strings.Contains(body, "eth_getBalance") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "error": map[string]interface{}{ + "code": -32000, + "message": "execution reverted", + }, + }) + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.NotNil(t, resp["error"], "should forward upstream error") + }) +} + +// +// --- Tests: multiple independent connections --- +// + +func TestWebSocket_MultipleConnections(t *testing.T) { + // Verifies multiple simultaneous connections each get correct responses + t.Run("IndependentConnections", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn1 := dialWs(t, addr) + defer conn1.Close() + conn2 := dialWs(t, addr) + defer conn2.Close() + conn3 := dialWs(t, addr) + defer conn3.Close() + + resp1 := sendAndReceive(t, conn1, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + resp2 := sendAndReceive(t, conn2, `{"jsonrpc":"2.0","id":2,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + resp3 := sendAndReceive(t, conn3, `{"jsonrpc":"2.0","id":3,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + + assert.Equal(t, float64(1), resp1["id"]) + assert.Equal(t, float64(2), resp2["id"]) + assert.Equal(t, float64(3), resp3["id"]) + assert.Equal(t, "0xabc123", resp1["result"]) + assert.Equal(t, "0xabc123", resp2["result"]) + assert.Equal(t, "0xabc123", resp3["result"]) + }) + + // Verifies closing one connection does not affect others + t.Run("OneDisconnectDoesNotAffectOthers", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn1 := dialWs(t, addr) + conn2 := dialWs(t, addr) + defer conn2.Close() + + resp1 := sendAndReceive(t, conn1, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", resp1["result"]) + + conn1.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + conn1.Close() + time.Sleep(100 * time.Millisecond) + + resp2 := sendAndReceive(t, conn2, `{"jsonrpc":"2.0","id":2,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", resp2["result"]) + }) +} + +// +// --- Tests: subscriptions (eth_subscribe / eth_unsubscribe) --- +// + +func TestWebSocket_Subscriptions(t *testing.T) { + // Verifies the full subscribe -> receive notification -> unsubscribe lifecycle + t.Run("SubscribeReceiveUnsubscribe", func(t *testing.T) { + notifCh := make(chan struct{}, 1) + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_subscribe": + subId := "0xdeadbeef12345678" + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": subId}) + + go func() { + time.Sleep(200 * time.Millisecond) + mockWriteJSON(conn, map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": map[string]interface{}{ + "subscription": subId, + "result": map[string]interface{}{"number": "0x101", "hash": "0xaaa"}, + }, + }) + select { + case notifCh <- struct{}{}: + default: + } + }() + case "eth_unsubscribe": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": true}) + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + defer cleanup() + + time.Sleep(2 * time.Second) + + conn := dialWs(t, addr) + defer conn.Close() + + // Subscribe + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + assert.NotNil(t, resp["result"], "should return subscription ID") + assert.Nil(t, resp["error"], "should not have error") + clientSubId, ok := resp["result"].(string) + require.True(t, ok, "subscription ID should be a string") + assert.True(t, strings.HasPrefix(clientSubId, "0x"), "subscription ID should start with 0x") + + // Wait for notification from upstream + select { + case <-notifCh: + case <-time.After(5 * time.Second): + t.Fatal("timeout waiting for upstream to send notification") + } + + // Read the notification delivered to the client + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, notifMsg, err := conn.ReadMessage() + require.NoError(t, err, "should receive notification") + + var notif map[string]interface{} + require.NoError(t, json.Unmarshal(notifMsg, ¬if)) + assert.Equal(t, "eth_subscription", notif["method"]) + params, ok := notif["params"].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, clientSubId, params["subscription"], "notification should use client subscription ID") + + // Unsubscribe + unsubResp := sendAndReceive(t, conn, fmt.Sprintf(`{"jsonrpc":"2.0","id":2,"method":"eth_unsubscribe","params":["%s"]}`, clientSubId)) + assert.Equal(t, true, unsubResp["result"]) + }) + + // Verifies eth_subscribe returns an error when no WS upstream is configured + t.Run("SubscribeWithNoWsUpstreamFails", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + assert.NotNil(t, resp["error"], "should return error when no WS upstream available") + }) + + // Verifies eth_unsubscribe with a nonexistent ID returns an error + t.Run("UnsubscribeUnknownIDFails", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_unsubscribe","params":["0xnonexistent"]}`) + assert.NotNil(t, resp["error"], "should return error for unknown subscription ID") + }) + + // Verifies client disconnect cleanly removes the client from the fan-out + // group without disturbing the persistent upstream newHeads subscription. + t.Run("SubscriptionCleanupOnDisconnect", func(t *testing.T) { + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_subscribe": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0xsub123"}) + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + defer cleanup() + + time.Sleep(2 * time.Second) + + conn := dialWs(t, addr) + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp["result"]) + + // Disconnect without unsubscribing — newHeads upstream sub stays active + conn.Close() + + // Allow cleanup to complete + time.Sleep(500 * time.Millisecond) + + // Verify a new client can still subscribe (upstream sub still alive) + conn2 := dialWs(t, addr) + defer conn2.Close() + + resp2 := sendAndReceive(t, conn2, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + assert.NotNil(t, resp2["result"], "new client should be able to subscribe after previous client disconnected") + }) +} + +// +// --- Tests: WebSocket upstream client --- +// + +func TestWebSocket_UpstreamClient(t *testing.T) { + // Verifies regular RPC requests can be routed through a WS upstream + t.Run("RPCThroughWsUpstream", func(t *testing.T) { + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_getBalance": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0xws_upstream_balance"}) + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + + // Config with ONLY the WS upstream to ensure requests go through WS + cfg := &common.Config{ + Server: &common.ServerConfig{ + ListenV4: util.BoolPtr(true), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_ws", + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Id: "ws-only", + Type: common.UpstreamTypeEvm, + Endpoint: wsUpstreamURL, + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } + + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + time.Sleep(2 * time.Second) + + httpURL := fmt.Sprintf("http://%s/test_ws/evm/123", addr) + cleanClient := &http.Client{Transport: &http.Transport{}} + httpResp, err := cleanClient.Post(httpURL, "application/json", + strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`)) + require.NoError(t, err) + defer httpResp.Body.Close() + + var resp map[string]interface{} + require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&resp)) + assert.Equal(t, "0xws_upstream_balance", resp["result"], "response should come from WS upstream") + }) + + // Verifies the WS upstream client automatically reconnects after disconnect + t.Run("WsUpstreamReconnects", func(t *testing.T) { + connCount := 0 + var connMu sync.Mutex + + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + connMu.Lock() + connCount++ + count := connCount + connMu.Unlock() + + if count == 1 { + // First connection: accept one request then close abruptly + _, msg, err := conn.ReadMessage() + if err != nil { + return + } + var req map[string]interface{} + json.Unmarshal(msg, &req) + id := req["id"] + method, _ := req["method"].(string) + if method == "eth_chainId" { + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x7b"}) + } + time.Sleep(100 * time.Millisecond) + conn.Close() + return + } + + // Subsequent connections: handle normally + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_getBalance": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0xreconnected"}) + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + defer cleanup() + + time.Sleep(5 * time.Second) + + connMu.Lock() + assert.GreaterOrEqual(t, connCount, 2, "WS upstream should have reconnected") + connMu.Unlock() + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.NotNil(t, resp["result"], "should get a response after upstream reconnect") + }) +} + +// +// --- Tests: subscription recovery after upstream reconnect --- +// + +func TestWebSocket_SubscriptionRecovery(t *testing.T) { + // Verifies that eth_subscribe returns an error when the only WS upstream + // never initialises (it closes every connection before answering). + t.Run("SubscribeFailsWhenWsUpstreamNeverInitialises", func(t *testing.T) { + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + conn.Close() + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + defer cleanup() + + time.Sleep(2 * time.Second) + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + assert.NotNil(t, resp["error"], "should return error when upstream WS is not connected") + t.Logf("got expected error: %v", resp["error"]) + }) +} + +// +// --- Tests: subscription deduplication --- +// + +func TestWebSocket_SubscriptionDedup(t *testing.T) { + // Verifies two clients subscribing to the same event share one upstream subscription + t.Run("TwoClientsShareOneUpstreamSubscription", func(t *testing.T) { + subscribeCount := 0 + var subMu sync.Mutex + upstreamSubId := "0xsharedsub123" + + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_subscribe": + subMu.Lock() + subscribeCount++ + count := subscribeCount + subMu.Unlock() + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": upstreamSubId}) + + // Send a notification only on first subscribe to avoid duplicates + if count == 1 { + go func() { + time.Sleep(500 * time.Millisecond) + mockWriteJSON(conn, map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": map[string]interface{}{ + "subscription": upstreamSubId, + "result": map[string]interface{}{"number": "0x999"}, + }, + }) + }() + } + case "eth_unsubscribe": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": true}) + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + defer cleanup() + time.Sleep(2 * time.Second) + + // Client 1 subscribes + conn1 := dialWs(t, addr) + defer conn1.Close() + resp1 := sendAndReceive(t, conn1, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp1["result"]) + clientSubId1 := resp1["result"].(string) + + // Client 2 subscribes to the same event + conn2 := dialWs(t, addr) + defer conn2.Close() + resp2 := sendAndReceive(t, conn2, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp2["result"]) + clientSubId2 := resp2["result"].(string) + + assert.NotEqual(t, clientSubId1, clientSubId2, "each client should get a unique subscription ID") + + // Only ONE eth_subscribe should have been sent to the upstream (dedup) + subMu.Lock() + assert.Equal(t, 1, subscribeCount, "upstream should only receive one eth_subscribe (dedup)") + subMu.Unlock() + + // Both clients should receive the notification + conn1.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, notif1, err := conn1.ReadMessage() + require.NoError(t, err) + + conn2.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, notif2, err := conn2.ReadMessage() + require.NoError(t, err) + + var n1, n2 map[string]interface{} + require.NoError(t, json.Unmarshal(notif1, &n1)) + require.NoError(t, json.Unmarshal(notif2, &n2)) + + p1 := n1["params"].(map[string]interface{}) + p2 := n2["params"].(map[string]interface{}) + assert.Equal(t, clientSubId1, p1["subscription"]) + assert.Equal(t, clientSubId2, p2["subscription"]) + }) +} + +// +// --- Tests: rate limiting --- +// + +func TestWebSocket_RateLimiting(t *testing.T) { + // Verifies project-level rate limits are enforced on WebSocket requests + t.Run("ProjectRateLimitAppliedOverWs", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := &common.Config{ + Server: &common.ServerConfig{ + ListenV4: util.BoolPtr(true), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_ws", + RateLimitBudget: "ws-test-budget", + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{ + Store: &common.RateLimitStoreConfig{ + Driver: "memory", + }, + Budgets: []*common.RateLimitBudgetConfig{ + { + Id: "ws-test-budget", + Rules: []*common.RateLimitRuleConfig{ + { + Method: "*", + MaxCount: 3, + Period: common.RateLimitPeriodMinute, + }, + }, + }, + }, + }, + } + + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + // No need to align to the minute: at most one window rollover fits in + // this loop, which still allows only 2*MaxCount of the 10 requests. + var lastResp map[string]interface{} + rateLimited := false + for i := 0; i < 10; i++ { + msg := fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"eth_getBalance","params":["0xaaaa","latest"]}`, i) + err := conn.WriteMessage(websocket.TextMessage, []byte(msg)) + require.NoError(t, err) + + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, respBytes, err := conn.ReadMessage() + require.NoError(t, err) + + require.NoError(t, json.Unmarshal(respBytes, &lastResp)) + if lastResp["error"] != nil { + errStr, _ := json.Marshal(lastResp["error"]) + if strings.Contains(string(errStr), "RateLimitRuleExceeded") || + strings.Contains(string(errStr), "rate limit") || + strings.Contains(string(errStr), "ErrProjectRateLimitRuleExceeded") { + rateLimited = true + break + } + } + } + + assert.True(t, rateLimited, "should hit rate limit when sending requests over WS") + }) +} + +// +// --- Tests: graceful shutdown --- +// + +func TestWebSocket_GracefulShutdown(t *testing.T) { + // Verifies clients receive a GoingAway close frame on server shutdown + t.Run("ServerShutdownClosesWsWithGoingAway", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", resp["result"]) + + cleanup() + + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, _, err := conn.ReadMessage() + var closeErr *websocket.CloseError + require.ErrorAs(t, err, &closeErr) + assert.Equal(t, websocket.CloseGoingAway, closeErr.Code, + "should receive CloseGoingAway (1001) on server shutdown") + }) + + // Verifies the server unsubscribes from upstreams during shutdown + // Verifies server shutdown closes the upstream WS connection (which implicitly + // terminates all upstream subscriptions) and closes all client connections. + t.Run("ServerShutdownCleansUpSubscriptions", func(t *testing.T) { + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_subscribe": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0xshutdownsub"}) + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + time.Sleep(2 * time.Second) + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp["result"]) + + cleanup() + + // After shutdown, the client connection should be closed + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, _, err := conn.ReadMessage() + var closeErr *websocket.CloseError + require.ErrorAs(t, err, &closeErr) + assert.Equal(t, websocket.CloseGoingAway, closeErr.Code) + }) + + // Verifies all connections are closed when the server shuts down + t.Run("MultipleConnectionsClosedOnShutdown", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + + conn1 := dialWs(t, addr) + defer conn1.Close() + conn2 := dialWs(t, addr) + defer conn2.Close() + conn3 := dialWs(t, addr) + defer conn3.Close() + + resp1 := sendAndReceive(t, conn1, `{"jsonrpc":"2.0","id":1,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", resp1["result"]) + resp2 := sendAndReceive(t, conn2, `{"jsonrpc":"2.0","id":2,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", resp2["result"]) + resp3 := sendAndReceive(t, conn3, `{"jsonrpc":"2.0","id":3,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", resp3["result"]) + + cleanup() + + for _, conn := range []*websocket.Conn{conn1, conn2, conn3} { + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, _, err := conn.ReadMessage() + var closeErr *websocket.CloseError + require.ErrorAs(t, err, &closeErr) + assert.Equal(t, websocket.CloseGoingAway, closeErr.Code) + } + }) +} + +// +// --- Tests: method filtering --- +// + +func TestWebSocket_MethodFiltering(t *testing.T) { + // Verifies ignored methods return an unsupported error over WebSocket + t.Run("IgnoredMethodReturnsError", func(t *testing.T) { + setupGock() + defer util.ResetGock() + + cfg := &common.Config{ + Server: &common.ServerConfig{ + ListenV4: util.BoolPtr(true), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_ws", + IgnoreMethods: []string{"debug_*"}, + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } + + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"debug_traceTransaction","params":["0xabc"]}`) + assert.NotNil(t, resp["error"], "ignored method should return error") + errObj := resp["error"].(map[string]interface{}) + assert.Contains(t, errObj["message"], "not supported") + }) +} + +// TestWebSocket_FailedSubscribeKeepsConnectionOpen: a failed subscribe must +// not close the connection and its other subscriptions. +func TestWebSocket_FailedSubscribeKeepsConnectionOpen(t *testing.T) { + setupGock() + defer util.ResetGock() + + addr, cleanup := setupTestERPCServer(t, httpOnlyConfig()) + defer cleanup() + + conn := dialWs(t, addr) + defer conn.Close() + + // Subscribe MUST fail (no WS upstream configured) but the connection + // must remain open for subsequent requests. + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp["error"], "subscribe should fail") + + // Connection should still work — send a normal RPC after failed subscribe. + conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + resp2 := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":2,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.Equal(t, "0xabc123", resp2["result"], "connection should remain open after subscribe failure") +} + +// TestWebSocket_EveryUpstreamHeadReachesItsPoller: a head delivered by +// several upstreams updates every one of their state pollers, though the +// client receives it once. +func TestWebSocket_EveryUpstreamHeadReachesItsPoller(t *testing.T) { + // Two WS upstreams that both deliver the same block. + upSubId := "0xupsub" + + deliverNotification := func(conn *websocket.Conn, blockHash string) { + mockWriteJSON(conn, map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": map[string]interface{}{ + "subscription": upSubId, + "result": map[string]interface{}{ + "number": "0x123", + "hash": blockHash, + }, + }, + }) + } + + makeMock := func() *httptest.Server { + return mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_subscribe": + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": upSubId}) + go func() { + time.Sleep(300 * time.Millisecond) + deliverNotification(conn, "0xsamehash") + }() + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + }) + }) + } + mock1 := makeMock() + defer mock1.Close() + mock2 := makeMock() + defer mock2.Close() + + setupGock() + defer util.ResetGock() + + url1 := "ws" + strings.TrimPrefix(mock1.URL, "http") + url2 := "ws" + strings.TrimPrefix(mock2.URL, "http") + srv, addr, cancel := startTestERPCServer(t, multiWsConfig(url1, url2)) + defer func() { _ = srv.Shutdown(&log.Logger); cancel() }() + time.Sleep(2 * time.Second) + + conn := dialWs(t, addr) + defer conn.Close() + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp["result"], "subscribe should succeed with multiple WS upstreams") + + // Both upstreams delivered block 0x123, so both pollers must know it. + prj, err := srv.erpc.GetProject("test_ws") + require.NoError(t, err) + require.Eventually(t, func() bool { + seen := 0 + for _, up := range prj.upstreamsRegistry.GetNetworkUpstreams(context.Background(), "evm:123") { + if strings.HasPrefix(up.Id(), "ws-upstream-") && up.EvmStatePoller().LatestBlock() >= 0x123 { + seen++ + } + } + return seen == 2 + }, 3*time.Second, 20*time.Millisecond, "every delivering upstream's poller must see the block") + + // Client should receive exactly one fan-out (deduped across upstreams). + conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err, "should receive deduped notification") + var notif map[string]interface{} + require.NoError(t, json.Unmarshal(msg, ¬if)) + assert.Equal(t, "eth_subscription", notif["method"]) + + // A second notification should NOT arrive — dedup must drop the duplicate + // from the second upstream. + conn.SetReadDeadline(time.Now().Add(800 * time.Millisecond)) + _, _, err = conn.ReadMessage() + assert.Error(t, err, "duplicate block from second upstream must be deduped") +} + +// TestWebSocket_LogsSubscribeOnAllUpstreamsAndDedupe: a logs subscription +// is made on every WS upstream and its notifications are deduplicated by +// blockHash + txHash + logIndex. +func TestWebSocket_LogsSubscribeOnAllUpstreamsAndDedupe(t *testing.T) { + subscribeCount := int64(0) + logSubId := "0xlogsub" + + deliverLog := func(conn *websocket.Conn, blockHash, txHash, logIndex string) { + mockWriteJSON(conn, map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscription", + "params": map[string]interface{}{ + "subscription": logSubId, + "result": map[string]interface{}{ + "address": "0xabc", + "blockHash": blockHash, + "transactionHash": txHash, + "logIndex": logIndex, + "removed": false, + "topics": []string{}, + "data": "0x", + "blockNumber": "0x123", + "transactionIndex": "0x0", + }, + }, + }) + } + + makeMock := func() *httptest.Server { + return mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + switch method { + case "eth_subscribe": + params, _ := req["params"].([]interface{}) + if len(params) > 0 && params[0] == "logs" { + atomic.AddInt64(&subscribeCount, 1) + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": logSubId}) + go func() { + time.Sleep(300 * time.Millisecond) + // Both upstreams deliver the same log. + deliverLog(conn, "0xblock1", "0xtx1", "0x0") + }() + } else { + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0xothersub"}) + } + default: + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + } + }) + }) + } + mock1 := makeMock() + defer mock1.Close() + mock2 := makeMock() + defer mock2.Close() + + setupGock() + defer util.ResetGock() + + url1 := "ws" + strings.TrimPrefix(mock1.URL, "http") + url2 := "ws" + strings.TrimPrefix(mock2.URL, "http") + addr, cleanup := setupTestERPCServer(t, multiWsConfig(url1, url2)) + defer cleanup() + time.Sleep(2 * time.Second) + + conn := dialWs(t, addr) + defer conn.Close() + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["logs",{}]}`) + require.NotNil(t, resp["result"]) + + // Wait for both upstreams to be subscribed. + require.Eventually(t, func() bool { + return atomic.LoadInt64(&subscribeCount) >= 2 + }, 5*time.Second, 100*time.Millisecond, "both WS upstreams should be subscribed for logs") + + // Client should receive exactly one log notification (deduped). + conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + _, msg, err := conn.ReadMessage() + require.NoError(t, err, "should receive deduped log notification") + var notif map[string]interface{} + require.NoError(t, json.Unmarshal(msg, ¬if)) + assert.Equal(t, "eth_subscription", notif["method"]) + + // Duplicate from second upstream must be deduped. + conn.SetReadDeadline(time.Now().Add(800 * time.Millisecond)) + _, _, err = conn.ReadMessage() + assert.Error(t, err, "duplicate log from second upstream must be deduped by blockHash+txHash+logIndex") +} + +// TestWebSocket_ReconnectAfterUnsubscribe: an upstream reconnect after the +// last logs subscriber left must not touch the released filter. +func TestWebSocket_ReconnectAfterUnsubscribe(t *testing.T) { + logSubId := "0xunsublog" + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + if method == "eth_subscribe" { + params, _ := req["params"].([]interface{}) + if len(params) > 0 && params[0] == "logs" { + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": logSubId}) + return + } + } + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0xothersub"}) + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + defer cleanup() + time.Sleep(2 * time.Second) + + conn := dialWs(t, addr) + defer conn.Close() + + // Subscribe to logs (creates filter sub group). + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["logs",{}]}`) + require.NotNil(t, resp["result"]) + clientSubId := resp["result"].(string) + + // Unsubscribe (tears down the filter group). + unsubResp := sendAndReceive(t, conn, fmt.Sprintf(`{"jsonrpc":"2.0","id":2,"method":"eth_unsubscribe","params":["%s"]}`, clientSubId)) + assert.Equal(t, true, unsubResp["result"]) + + // Force the upstream WS to drop so it reconnects after the release. + mockUpstream.CloseClientConnections() + + // Sleep long enough for reconnect + callback to fire. + time.Sleep(3 * time.Second) + + // eRPC should still be alive — make a normal RPC call. + resp2 := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":3,"method":"eth_getBalance","params":["0xaaaa","latest"]}`) + assert.NotNil(t, resp2, "eRPC should not have crashed from reconnect-after-unsubscribe") +} + +// TestWebSocket_InternalRequestIdsAreUnique: internal eth_subscribe requests +// use unique wire ids, so their responses cannot be misrouted. +func TestWebSocket_InternalRequestIdsAreUnique(t *testing.T) { + subscribeIds := make([]int64, 0) + var idMu sync.Mutex + + mockUpstream := mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, req map[string]interface{}) { + if method == "eth_subscribe" { + idMu.Lock() + if f, ok := id.(float64); ok { + subscribeIds = append(subscribeIds, int64(f)) + } + idMu.Unlock() + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0xabc"}) + return + } + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": "0x1"}) + }) + }) + defer mockUpstream.Close() + + setupGock() + defer util.ResetGock() + + wsUpstreamURL := "ws" + strings.TrimPrefix(mockUpstream.URL, "http") + addr, cleanup := setupTestERPCServer(t, standardWsConfig(wsUpstreamURL)) + defer cleanup() + time.Sleep(2 * time.Second) + + // Trigger an internal upstream subscribe via a client subscribe. The + // upstream eth_subscribe is dispatched asynchronously, so poll for it. + conn := dialWs(t, addr) + defer conn.Close() + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.NotNil(t, resp["result"]) + + require.Eventually(t, func() bool { + idMu.Lock() + defer idMu.Unlock() + return len(subscribeIds) >= 1 + }, 3*time.Second, 20*time.Millisecond, "should have observed at least one internal eth_subscribe") + + idMu.Lock() + defer idMu.Unlock() + + // All IDs unique. + seen := make(map[int64]bool) + for _, id := range subscribeIds { + assert.False(t, seen[id], "internal id %d duplicated", id) + seen[id] = true + } +} From 00d791ab70232bae5ccbc2a383da294e2f592c20 Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 13:28:45 +0100 Subject: [PATCH 08/21] docs: WebSocket, failover and tip-consistency documentation Co-Authored-By: Claude Opus 5.5 --- .../config/database/evm-json-rpc-cache.mdx | 2 + docs/pages/config/database/shared-state.mdx | 17 +- docs/pages/config/failsafe/integrity.mdx | 9 +- docs/pages/config/projects/networks.mdx | 98 +++++ docs/pages/config/projects/upstreams.mdx | 6 +- docs/pages/operation/_meta.js | 3 + docs/pages/operation/production.mdx | 11 +- docs/pages/operation/url.mdx | 10 + docs/pages/operation/websocket.mdx | 345 ++++++++++++++++++ docs/pages/reference/metrics.mdx | 3 + 10 files changed, 489 insertions(+), 15 deletions(-) create mode 100644 docs/pages/operation/websocket.mdx diff --git a/docs/pages/config/database/evm-json-rpc-cache.mdx b/docs/pages/config/database/evm-json-rpc-cache.mdx index 56db58025..be32b8961 100644 --- a/docs/pages/config/database/evm-json-rpc-cache.mdx +++ b/docs/pages/config/database/evm-json-rpc-cache.mdx @@ -192,6 +192,8 @@ All fields are under `database.evmJsonRpcCache`. `Duration` values accept Go dur |---|---|---|---| | `database.evmJsonRpcCache` | `*CacheConfig` | `nil` | When `nil` (omitted): cache subsystem is entirely disabled — the `EvmJsonRpcCache` object is never created. `SetDefaults` on sub-fields is only called when this pointer is non-nil. The auto-generated project (`id: "main"`, created when no `projects:` block exists) has NO `evmJsonRpcCache` config. You must provide the key explicitly to enable caching. Source: | +For requests that keep a moving tag untranslated (`"latest"`, `"finalized"` — by default `eth_getBlockByNumber`), eRPC keys the cache entry by the concrete block number — the response's own number on write, the network's tracked tip on read. `"safe"` keeps its literal-tag key. That means a burst of concurrent tag queries landing on the same tip coalesces onto one cached entry, but once the tip advances the key changes and the next request forwards upstream. You don't have to worry about a `"latest"` query returning a block that's multiple tips stale just because the TTL hasn't expired. + #### `evmJsonRpcCache.compression` Auto-created even when omitted: `CacheConfig.SetDefaults` always creates `&CompressionConfig{}` and calls its `SetDefaults()` when `c.Compression == nil`. The only way to opt out is `compression.enabled: false`. diff --git a/docs/pages/config/database/shared-state.mdx b/docs/pages/config/database/shared-state.mdx index db301b0f8..c85db3756 100644 --- a/docs/pages/config/database/shared-state.mdx +++ b/docs/pages/config/database/shared-state.mdx @@ -102,7 +102,7 @@ https://docs.erpc.cloud/config/database/shared-state.llms.txt`} ### How it works -**Registry and key space.** `NewSharedStateRegistry` wraps one `Connector` and a `clusterKey` prefix. Every counter is stored under `//` — for example `prod-main/latestBlock/projectX/evm/1/alchemy`. The registry holds a `sync.Map` of live `counterInt64` instances; calling `GetCounterInt64` with the same key always returns the same object (idempotent `LoadOrStore`). On first call, two async bootstrap tasks start: one seeds the local counter with a single remote `Get`, and one calls `WatchCounterInt64` to open a long-lived change stream from the connector. +**Registry and key space.** `NewSharedStateRegistry` wraps one `Connector` and a `clusterKey` prefix. Every counter is stored under `/` — for example `prod-main/v2/latestBlock//` (see **Counter families** below). The registry holds a `sync.Map` of live `counterInt64` instances; calling `GetCounterInt64` with the same key always returns the same object (idempotent `LoadOrStore`). On first call, two async bootstrap tasks start: one seeds the local counter with a single remote `Get`, and one calls `WatchCounterInt64` to open a long-lived change stream from the connector. **Write path.** `TryUpdate` is synchronous and local-only — it atomically advances the in-memory counter via a CAS loop and then schedules a background push without touching the network. `scheduleBackgroundPushCurrent` coalesces concurrent pushes: a single goroutine per counter (1) publishes the current state via `connector.PublishCounterInt64` for fast peer notification, (2) acquires the distributed lock within `lockMaxWait`, (3) reads remote state and adopts it if the remote value is newer, then writes local state if local is newer. If the lock cannot be acquired within `lockMaxWait`, the goroutine falls back to publish-only propagation and reschedules if another update arrived while it was running. @@ -114,7 +114,7 @@ https://docs.erpc.cloud/config/database/shared-state.llms.txt`} **Transport differences by connector.** All five connectors implement the same `GetCounterInt64` / `PublishCounterInt64` / `WatchCounterInt64` interface; propagation latency differs significantly: -- **Redis** — push via `PUBLISH counter:` (e.g. `counter:prod-main/latestBlock/...`) and subscribe via a single shared `PSubscribe("counter:*")` connection that matches all counters regardless of cluster; subscribers filter by exact key inside `runMessageLoop`. Reconnects automatically with exponential backoff (1 s → 30 s with jitter) and a 5-minute periodic poll fallback; `pollAllKeys()` is triggered after each reconnect to close the gap during the reconnection window. Subscriber channels are buffered (size 1, drop-on-full) so slow consumers never block the message loop. Source: , +- **Redis** — push via `PUBLISH counter:` (e.g. `counter:prod-main/v2/latestBlock//`) and subscribe via a single shared `PSubscribe("counter:*")` connection that matches all counters regardless of cluster; subscribers filter by exact key inside `runMessageLoop`. Reconnects automatically with exponential backoff (1 s → 30 s with jitter) and a 5-minute periodic poll fallback; `pollAllKeys()` is triggered after each reconnect to close the gap during the reconnection window. Subscriber channels are buffered (size 1, drop-on-full) so slow consumers never block the message loop. Source: , - **PostgreSQL** — push via `pg_notify` using a channel name derived by `sanitizeChannelName(fmt.Sprintf("counter_%s", key))` to strip characters illegal in PostgreSQL NOTIFY channel names; LISTEN-based subscription with a 30 s polling fallback. Source: - **DynamoDB** — no push primitive. `PublishCounterInt64` is a documented no-op; propagation relies on polling at `statePollInterval` (default 5 s). Cross-pod convergence latency is bounded by the poll interval, not a push event. Source: - **Memory** — both `WatchCounterInt64` and `PublishCounterInt64` are no-ops; in-process only, no cross-pod coordination. Source: @@ -122,14 +122,15 @@ https://docs.erpc.cloud/config/database/shared-state.llms.txt`} **Deadlock-prevention invariant.** The foreground paths (`TryUpdate` and `TryUpdateIfStale`) never acquire the distributed lock. Only the background `scheduleBackgroundPushCurrent` goroutine acquires it, and only after any local update is complete. This strict lock-ordering rule was introduced to fix a deadlock where the old design could have `scheduleBackgroundPushCurrent` hold the distributed lock and wait for `updateMu`, while `TryUpdateIfStale` held `updateMu` and waited for the distributed lock. The test `TestLockOrderSimulation_OldBuggyBehavior` documents the old behavior. -**Counter families.** Four families of counters are materialised during normal operation, all stored under `//<...>` in the backing store: +**Counter families.** These counters are materialised during normal operation, all stored under `/` in the backing store: -1. `latestBlock////` — latest block per upstream (`ignoreRollbackOf=1024`). -2. `finalizedBlock////` — finalized block per upstream (`ignoreRollbackOf=1024`). -3. `earliestBlock/////` — earliest available block per upstream per probe type (`ignoreRollbackOf=0`; all decreases accepted because node pruning genuinely shifts the historical anchor). -4. `servedLatestBlock//>` and `servedFinalizedBlock//>` — cross-pod monotonic served-tip per tag-group. +1. `v2/latestBlock/` — latest block per upstream (`ignoreRollbackOf=1024`). +2. `v2/finalizedBlock/` — finalized block per upstream (`ignoreRollbackOf=1024`). +3. `v2/earliestBlock//` — earliest available block per upstream per probe type (`ignoreRollbackOf=0`; all decreases accepted because node pruning genuinely shifts the historical anchor). +4. `v2/operatorCordon//` — operator cordon state per upstream. +5. `network///latestBlock` — the network's delivered-head floor: the highest head pushed to a WebSocket subscriber (see [WebSocket](/operation/websocket)). -The per-upstream key component is derived via `common.UniqueUpstreamKey(up)` which hashes the upstream's project/network/upstream triple to prevent collisions between upstreams with the same ID in different networks or projects. Source: +`` is `common.UniqueUpstreamKey(up)`: the upstream id followed by a hash of its id, endpoint and JSON-RPC headers. It does not depend on chain detection, so it stays the same for the upstream's lifetime. Two upstreams share a key only when they call the same endpoint with the same headers, and so report the same chain. Changing an upstream's endpoint or headers starts a new counter, and so does upgrading from a version that also hashed the network id; the old keys are left behind in the store. Source: **Instance ID.** Each counter state carries an `UpdatedBy` field (`"b"` in JSON) identifying which pod wrote it. It is resolved from the first non-empty value of: env `INSTANCE_ID` → env `POD_NAME` → env `HOSTNAME` → `os.Hostname()` → `"unknown"`. This is diagnostic-only and does not affect routing. Source: diff --git a/docs/pages/config/failsafe/integrity.mdx b/docs/pages/config/failsafe/integrity.mdx index 902e0471f..acc0446b8 100644 --- a/docs/pages/config/failsafe/integrity.mdx +++ b/docs/pages/config/failsafe/integrity.mdx @@ -137,10 +137,11 @@ request-aware: a `use-upstream` selector scopes it to the targeted subset. `); `pickHighestBlock` keeps the higher of the two — protecting against -a corrupted state pointer. Cache responses skip the re-fetch (the read-side realtime age -guard handles staleness instead). +eRPC re-requests that block (`SkipCacheRead=true`) — excluding the upstream that answered +(`UseUpstream=!`) unless the request already carries a `use-upstream` selector, which +is kept; `pickHighestBlock` keeps the higher of the two — protecting against a corrupted +state pointer. Cache hits are enforced the same way, so a cached block behind the tip is +re-fetched rather than served. **Stale-tip cache write guard.** Under `enforceHighestBlock`, a realtime-finality response already behind the network-wide tip is never written to cache (the classic diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 49104ddf2..0258fbdd2 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -51,6 +51,104 @@ Illustrative, not a tuned production config — declare Ethereum with a human-re }]`} /> +## `eth_subscribe` + +Some JSON-RPC clients include `fromBlock: "0x0"` in the `eth_subscribe("logs", {…})` filter as a "from genesis" marker. `eth_subscribe` is a live-stream RPC — `fromBlock` has no standardised meaning there — and on backends that prune historical data the subscribe call fails outright. Enable `stripSubscribeFromBlockZero` on the network to drop the field before forwarding so the live stream succeeds; historical logs stay retrievable via `eth_getLogs` in the normal way. + + + +```yaml filename="erpc.yaml" +projects: + - id: main + networks: + - architecture: evm + evm: + chainId: 1 + # Drops fromBlock: "0x0" / "0" from eth_subscribe logs filters + # before forwarding to upstream WebSockets. Only a value that + # parses to zero is stripped — other fromBlocks pass through. + stripSubscribeFromBlockZero: true # default: false +``` + + +```ts filename="erpc.ts" +import { createConfig } from "@erpc-cloud/config"; + +export default createConfig({ + projects: [ + { + id: "main", + networks: [ + { + architecture: "evm", + evm: { + chainId: 1, + // Drops fromBlock: "0x0" / "0" from eth_subscribe logs filters + // before forwarding to upstream WebSockets. Only a value that + // parses to zero is stripped — other fromBlocks pass through. + stripSubscribeFromBlockZero: true, // default: false + }, + }, + ], + }, + ], +}); +``` + + + +Leave unset for chains where upstreams handle `fromBlock: "0x0"` in an `eth_subscribe` filter correctly (most do — the flag is a targeted workaround, not a universal default). It can also be set once for every network under `networkDefaults.evm`. + +## Failover + +By default a request only reaches the upstreams the [selection policy](/config/projects/selection-policies) currently routes to; upstreams tagged `tier:fallback` join once the policy promotes them, which happens on its evaluation interval. `failover.onDefaultsExhausted` adds a per-request escape for the moments before that: when every routed upstream failed with an error that another upstream might not repeat, the same request is retried once on the `tier:fallback` upstreams it has not tried yet — even while the policy still keeps them cordoned. + + + +```yaml filename="erpc.yaml" +projects: + - id: main + networkDefaults: + failover: + onDefaultsExhausted: true # default: false + upstreams: + - endpoint: https://primary.example.com + - endpoint: https://backup.example.com + tags: ["tier:fallback"] +``` + + +```ts filename="erpc.ts" +import { createConfig } from "@erpc-cloud/config"; + +export default createConfig({ + projects: [ + { + id: "main", + networkDefaults: { + failover: { onDefaultsExhausted: true }, // default: false + }, + upstreams: [ + { endpoint: "https://primary.example.com" }, + { endpoint: "https://backup.example.com", tags: ["tier:fallback"] }, + ], + }, + ], +}); +``` + + + +The escape: + +- escalates only on errors that are retryable toward the network — an error every upstream would return (e.g. an execution revert) is returned as-is; +- treats an empty result as a miss only for a concrete block number the network should already have (see `evm.emptyResultConfidence`), so polling for a pending transaction receipt does not sweep the fallbacks; +- skips fallbacks whose circuit breaker is open for the method, or that don't handle the method; +- fires at most once per request, and never for consensus requests; +- is counted in `erpc_network_fallback_escape_total{project,network,category}` — expect zero in steady state. + +The validation report warns when `onDefaultsExhausted` is enabled but no upstream is tagged `tier:fallback`. + ## Agent reference Copy one of these prompts into your AI agent session (Claude Code, Cursor, …) — each one diff --git a/docs/pages/config/projects/upstreams.mdx b/docs/pages/config/projects/upstreams.mdx index 95e9d402b..55089f951 100644 --- a/docs/pages/config/projects/upstreams.mdx +++ b/docs/pages/config/projects/upstreams.mdx @@ -26,7 +26,9 @@ Illustrative, not a tuned production config — a minimal upstream definition: yaml={`projects: - id: main upstreams: - - # chain ID auto-detected; state poller starts automatically + - # chain ID auto-detected; state poller starts automatically. + # http(s) and ws(s) schemes are supported; use ws:// or wss:// + # for WebSocket upstreams (required for eth_subscribe support). endpoint: https://eth-mainnet.g.alchemy.com/v2/YOUR_KEY`} ts={`projects: [{ id: "main", @@ -202,7 +204,7 @@ All paths relative to `projects[*]`. "Default" = value after `SetDefaults`/`Appl |---|---|---|---| | `upstreams[*].id` | string | Derived: native-scheme endpoint → `-`; non-native → `-`; if `vendorName` set → `-` () | Used in task names, metrics labels, `use-upstream` matching, admin cordon RPCs. | | `upstreams[*].type` | `UpstreamType` | `"evm"` when omitted () | `evm` and `svm` are both supported at runtime. The `evm+` shorthand schemes are normalized to `"evm"` at defaults time. **Footgun**: `svm` is never inferred — an upstream serving Solana that omits `type` silently becomes an EVM upstream and fails to bootstrap. Always write `type: svm` explicitly. | -| `upstreams[*].endpoint` | string | `""` (required) | Native schemes: `http://`, `https://`, `grpc://`, `grpc+bds://`. Non-native (e.g. `alchemy://KEY`) converts the upstream into a provider. `ws://`/`wss://` pass validation but fail at client creation with "websocket client not implemented yet" — retries forever. Redacted in JSON/YAML output. | +| `upstreams[*].endpoint` | string | `""` (required) | Native schemes: `http://`, `https://`, `ws://`, `wss://`, `grpc://`, `grpc+bds://`. Non-native (e.g. `alchemy://KEY`) converts the upstream into a provider. `ws://`/`wss://` create a persistent WebSocket client (auto-reconnect, keepalive) that serves regular requests and `eth_subscribe` — see [WebSocket](/operation/websocket). Redacted in JSON/YAML output. | | `upstreams[*].tags` | `[]string` | nil; all-or-nothing inheritance from `upstreamDefaults.tags` () | `:` convention. Matched by `use-upstream` and policy stdlib. **Footgun**: one tag on the upstream drops ALL defaults tags. | | `upstreams[*].vendorName` | string | `""`; at runtime filled by URL pattern match or `guessVendorName()` | When non-empty, forces name-lookup only — `OwnsUpstream` URL matching is skipped. Mismatch silently applies the wrong error normalizer. No warning is logged. | | `upstreams[*].ignoreMethods` | `[]string` | nil; **forced to `["*"]` when `allowMethods` set and `ignoreMethods` nil** () | Evaluated first in `ShouldHandleMethod`. Glob wildcards + `\|` OR + `&` AND + `!` NOT. Result cached per method forever. | diff --git a/docs/pages/operation/_meta.js b/docs/pages/operation/_meta.js index 7165dd26d..cc92eac95 100644 --- a/docs/pages/operation/_meta.js +++ b/docs/pages/operation/_meta.js @@ -2,6 +2,9 @@ module.exports = { "url": { title: "URL", }, + "websocket": { + title: "WebSocket", + }, "healthcheck": { title: "Healthcheck", }, diff --git a/docs/pages/operation/production.mdx b/docs/pages/operation/production.mdx index a9b7e2653..b38b8278c 100644 --- a/docs/pages/operation/production.mdx +++ b/docs/pages/operation/production.mdx @@ -118,7 +118,7 @@ points the agent at this page's machine-readable reference so it can do the work **Server timeouts.** Three fields govern the full request lifecycle: `server.maxTimeout` (default `150s`) is a custom `TimeoutHandler` wrapping the entire HTTP handler chain — it is mandatory and validation rejects `0`. `server.readTimeout` (default `30s`) covers header and body read; `server.writeTimeout` (default `120s`) covers response write. These are Go `http.Server` fields. A `maxTimeout` hit returns JSON-RPC `-32603` with HTTP 200 for POST requests. Integer YAML values are interpreted as **milliseconds** — `maxTimeout: 150` means 150 ms, not 150 s; always use the string form. -**Graceful drain (healthcheck probes).** On SIGTERM the shutdown sequence is: (1) `draining` flag flips immediately — healthcheck returns HTTP 503; (2) process sleeps `server.waitBeforeShutdown` (default `10s`) to let load-balancer drain the endpoint; (3) `http.Server.Shutdown` starts with a hardcoded 30 s budget; (4) after `appCtx` is done, `Init` sleeps `server.waitAfterShutdown` (default `10s`) before process exit. For Kubernetes with Cilium/Envoy, set both waits to `30s`. `terminationGracePeriodSeconds` must exceed `waitBeforeShutdown + waitAfterShutdown + expected drain time` or the process receives SIGKILL mid-drain. Sources: , , +**Graceful drain (healthcheck probes).** On SIGTERM the shutdown sequence is: (1) `draining` flag flips immediately — healthcheck returns HTTP 503 and client WebSocket connections are closed with `1001` once their in-flight requests answer; (2) process sleeps `server.waitBeforeShutdown` (default `10s`) to let load-balancer drain the endpoint; (3) `http.Server.Shutdown` starts with a hardcoded 30 s budget; (4) after `appCtx` is done, `Init` sleeps `server.waitAfterShutdown` (default `10s`) before process exit. For Kubernetes with Cilium/Envoy, set both waits to `30s`. `terminationGracePeriodSeconds` must exceed `waitBeforeShutdown + waitAfterShutdown + expected drain time` or the process receives SIGKILL mid-drain. Sources: , , **Trusted-IP forwarding.** When eRPC runs behind a load-balancer, `server.trustedIPForwarders` defaults to loopback only (`["127.0.0.1/8", "::1/128"]`). IP-based rate limits and `network` auth strategies apply to the proxy IP unless you add the LB CIDR to the forwarders list and name the forwarding header in `server.trustedIPHeaders`. The resolver walks headers in preference order, strips trailing trusted proxies from the XFF list right-to-left, and picks the nearest untrusted hop. @@ -424,6 +424,15 @@ metrics: { 16. **`erpc-server-pprof` is present in every image but is never the default CMD.** The pprof binary is always built into the image at `/erpc-server-pprof` but the default `CMD` is `/erpc-server`. Operators must override the container command to use pprof — it is not activated by any env var or config flag. Source: `Dockerfile:L84`, `Makefile` (`run-pprof` target) 17. **The `prometheus.yaml` at the repo root is NOT the monitoring container's config.** It is the production Prometheus Operator config (with `remote_write` to Mimir, in-cluster job discovery). The monitoring container at `monitoring/prometheus/prometheus.yml` is a separate file with different scrape targets. Loading the wrong one into the monitoring container will produce empty or incorrect dashboards. Source: `prometheus.yaml`, `monitoring/prometheus/prometheus.yml` +### WebSocket subscriptions + +If you serve [`eth_subscribe`](/operation/websocket) traffic, a few additional notes: + +- **One upstream subscription per (network, filter, upstream)** — eRPC de-duplicates across clients. 1,000 clients subscribed to the same `logs` filter on a network with N WebSocket upstreams consume N upstream subscriptions, not 1,000 × N. +- **Instance-local subscriptions** — a client WS connection lives on one eRPC instance; its subscriptions end when that connection closes, so clients must re-subscribe after reconnecting. +- **Slow-client policy** — each subscription gets a buffer of `server.webSocket.subscriptionBufferSize` notifications (default 256). `newHeads` / `newPendingTransactions` drop their own oldest notifications on overflow; `logs` subscriptions close the connection with `1013`. Watch `erpc_websocket_subscription_notifications_dropped_total`. +- **Shutdown** — client WS connections are closed with `1001` as soon as shutdown begins, so clients reconnect to another instance while the load balancer drains this one; `waitBeforeShutdown` only bounds how long their in-flight requests get to finish. + ### Observability | Metric | Type | Labels | When it fires | diff --git a/docs/pages/operation/url.mdx b/docs/pages/operation/url.mdx index b0804eb9a..c45fb84e0 100644 --- a/docs/pages/operation/url.mdx +++ b/docs/pages/operation/url.mdx @@ -41,6 +41,16 @@ Illustrative, not a tuned production config — map a dedicated domain to a sing }`} /> +## WebSocket + +You can also connect via WebSocket using the same URL path with a `ws://` or `wss://` scheme: + +``` +ws://localhost:4000/main/evm/1 +``` + +WebSocket connections support all JSON-RPC methods, batch requests, and real-time subscriptions (`eth_subscribe`). Read more in the [WebSocket](/operation/websocket) page. + ## Agent reference Copy one of these prompts into your AI agent session (Claude Code, Cursor, …) — each one diff --git a/docs/pages/operation/websocket.mdx b/docs/pages/operation/websocket.mdx new file mode 100644 index 000000000..d48d1e978 --- /dev/null +++ b/docs/pages/operation/websocket.mdx @@ -0,0 +1,345 @@ +--- +description: WebSocket support in eRPC for persistent connections and real-time subscriptions... +--- + +import { Callout, Tabs, Tab } from "nextra/components"; + +# WebSocket + +eRPC supports WebSocket connections on the same HTTP port. Clients can connect via `ws://` or `wss://` for persistent JSON-RPC connections and real-time subscriptions (`eth_subscribe`). + +## Connecting + +Connect to eRPC using the same URL path as HTTP, but with the `ws://` or `wss://` scheme: + +``` +ws:///// +``` + +```javascript +// Browser / Node.js example +const ws = new WebSocket("ws://localhost:4000/main/evm/1"); + +ws.onopen = () => { + ws.send(JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "eth_blockNumber", + params: [] + })); +}; + +ws.onmessage = (event) => { + console.log(JSON.parse(event.data)); + // { "jsonrpc": "2.0", "id": 1, "result": "0x1234567" } +}; +``` + + + WebSocket connections use the same port as HTTP — no additional configuration needed. Domain aliasing rules also apply to WebSocket connections. + + +## Regular JSON-RPC over WebSocket + +All JSON-RPC methods supported over HTTP also work over WebSocket. The connection is persistent, so you can send multiple requests without reconnecting: + +```javascript +// Send multiple requests on the same connection +ws.send(JSON.stringify({ jsonrpc: "2.0", id: 1, method: "eth_blockNumber", params: [] })); +ws.send(JSON.stringify({ jsonrpc: "2.0", id: 2, method: "eth_chainId", params: [] })); +ws.send(JSON.stringify({ jsonrpc: "2.0", id: 3, method: "eth_getBlockByNumber", params: ["latest", false] })); +``` + +Batch requests (JSON arrays) are also supported: + +```javascript +ws.send(JSON.stringify([ + { jsonrpc: "2.0", id: 1, method: "eth_blockNumber", params: [] }, + { jsonrpc: "2.0", id: 2, method: "eth_chainId", params: [] } +])); +// Response is a JSON array +``` + +All eRPC features work identically for WebSocket RPC requests: authentication, method filters, caching, upstream scoring, failsafe policies (retry, hedge, circuit breaker), rate limiting, multiplexing (request deduplication), EVM integrity checks, and metrics. Each request is bounded by `server.maxTimeout`, and a connection runs at most `server.webSocket.maxConcurrentRequestsPerConnection` requests at once — further messages wait until one finishes. + +## Subscriptions + +eRPC supports `eth_subscribe` and `eth_unsubscribe` for real-time event streaming. Subscriptions require at least one upstream configured with a `ws://` or `wss://` endpoint. + +### eth_subscribe + +```javascript +const ws = new WebSocket("ws://localhost:4000/main/evm/1"); + +ws.onopen = () => { + // Subscribe to new block headers + ws.send(JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "eth_subscribe", + params: ["newHeads"] + })); +}; + +ws.onmessage = (event) => { + const msg = JSON.parse(event.data); + + if (msg.id) { + // Subscription confirmation + console.log("Subscription ID:", msg.result); + // "0x2a151383161618ebffddfed9de4be5db" + } + + if (msg.method === "eth_subscription") { + // New block notification + console.log("New block:", msg.params.result.number); + } +}; +``` + +Supported subscription types (depends on upstream node capabilities): +- `newHeads` — new block headers +- `logs` — log entries matching a filter +- `newPendingTransactions` — pending transaction hashes + +#### Subscribing to logs with a filter + +The second params entry is a filter object with optional `address` (string or array) and `topics` (array of topic hex strings or arrays for OR-matching). The filter is hashed by eRPC so multiple clients requesting the same filter share a single underlying upstream subscription. + +```javascript +ws.send(JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "eth_subscribe", + params: [ + "logs", + { + address: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", // USDC + topics: [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" // Transfer + ] + } + ] +})); +``` + +### eth_unsubscribe + +```javascript +ws.send(JSON.stringify({ + jsonrpc: "2.0", + id: 2, + method: "eth_unsubscribe", + params: ["0x2a151383161618ebffddfed9de4be5db"] +})); +// Response: { "jsonrpc": "2.0", "id": 2, "result": true } +``` + +### Subscription features + +- **Resilient delivery** — `newHeads` is subscribed on every connected WebSocket upstream of the network, and each block is delivered from whichever upstream announces it first. `logs` and `newPendingTransactions` are subscribed on every WebSocket upstream whose circuit breaker is closed; with [`failover.onDefaultsExhausted`](/config/projects/networks#failover) enabled, upstreams tagged `tier:fallback` are used only when every other upstream rejects the subscribe. +- **No surprise disconnects** — upstream reconnects, scoring changes and brief outages never close your connection; eRPC re-subscribes upstream behind the scenes. eRPC closes a connection only on shutdown (`1001`), when the client stops answering pings, or when a `logs` subscription overflows its buffer (see below). +- **Deduplication** — even though multiple upstreams are subscribed in parallel, you only receive each block, log or pending tx once. Logs are identified by `blockHash`, `transactionHash` and `logIndex`; a log missing any of them is passed through without deduplication. +- **Shared subscriptions** — clients subscribing to the same event on the same network share one upstream subscription per upstream. Each client gets its own subscription ID and receives every notification independently. +- **Slow-client safety** — every subscription has its own buffer of `server.webSocket.subscriptionBufferSize` notifications, so a slow reader never blocks other clients or upstream delivery. When it overflows, `newHeads` and `newPendingTransactions` drop the oldest queued notification; a `logs` subscription closes the connection with `1013` instead, so the client knows it missed data and can re-subscribe and backfill with `eth_getLogs`. Drops are counted in `erpc_websocket_subscription_notifications_dropped_total`. +- **Connection-scoped IDs** — eRPC issues its own subscription IDs, so they stay the same across upstream reconnects and upstream switches. They belong to the connection that created them: `eth_unsubscribe` only works on that connection, and subscriptions end when it closes. +- **Rate limiting** — subscribe/unsubscribe calls count against the same project and network rate limits as regular RPC. + +### Reorg handling for `logs` subscriptions + +When a block containing one of your matched logs is reorged out, the upstream node re-sends that log with `"removed": true` on the same subscription. eRPC passes these notifications through unchanged, so you get the same semantics as a direct WebSocket connection to a node. If a log is removed and later re-added (the chain switches back), each transition is delivered once, even when several upstreams report it. + +```javascript +ws.onmessage = (event) => { + const msg = JSON.parse(event.data); + if (msg.method === "eth_subscription" && msg.params.result.removed) { + // This log was in a block that was reorged out. + // Roll back any state you derived from it. + } +}; +``` + +### When eth_subscribe fails + +`eth_subscribe` returns a JSON-RPC error when: + +- the network has no WebSocket upstream, or none of the selected upstreams is connected and accepts the subscription; +- the subscription type is not supported (`-32602`); +- the connection already has `server.webSocket.maxSubscriptionsPerConnection` active subscriptions; +- the request is rate limited. + +A failed `eth_subscribe` leaves your WebSocket connection open. You can retry on the same connection. + +## WebSocket upstreams + +To support subscriptions, configure at least one upstream with a `ws://` or `wss://` endpoint: + + + +```yaml filename="erpc.yaml" +projects: + - id: main + upstreams: + # Regular HTTP upstream (for standard RPC) + - id: eth-mainnet-http + endpoint: https://eth-mainnet.example.com + evm: + chainId: 1 + + # WebSocket upstream (enables subscriptions) + - id: eth-mainnet-ws + endpoint: ws://eth-mainnet.example.com:8546 + evm: + chainId: 1 + failsafe: + retry: + maxAttempts: 2 + delay: 100ms + timeout: + duration: 30s +``` + + +```ts filename="erpc.ts" +import { createConfig } from "@erpc-cloud/config"; + +export default createConfig({ + projects: [ + { + id: "main", + upstreams: [ + // Regular HTTP upstream + { + id: "eth-mainnet-http", + endpoint: "https://eth-mainnet.example.com", + evm: { chainId: 1 }, + }, + // WebSocket upstream (enables subscriptions) + { + id: "eth-mainnet-ws", + endpoint: "ws://eth-mainnet.example.com:8546", + evm: { chainId: 1 }, + failsafe: { + retry: { maxAttempts: 2, delay: "100ms" }, + timeout: { duration: "30s" }, + }, + }, + ], + }, + ], +}); +``` + + + + + WebSocket upstreams also serve regular JSON-RPC requests (not just subscriptions). eRPC's scoring system picks the best upstream regardless of transport — HTTP or WebSocket. + + +The WebSocket upstream client features: +- **Persistent connection** with automatic reconnection — every re-dial waits a jittered exponential backoff (1s to 30s), which resets only after a connection has stayed up for a full keepalive window +- **Ping/pong keepalive** (30s interval) to detect dead connections +- **Lazy connection** — if the upstream is unavailable at startup, eRPC retries in the background without blocking +- **Fail-fast while reconnecting** — requests sent while the connection is down fail immediately with a retryable error so they fail over to other upstreams + +`eth_subscribe` and `eth_unsubscribe` are only sent to upstreams by eRPC's own subscription handling. A plain HTTP `eth_subscribe` is never forwarded to a WebSocket upstream, since the resulting subscription would have no owner. + +## Server configuration + +WebSocket server settings can be tuned via the `webSocket` section in server config. Defaults work well for most deployments: + +```yaml filename="erpc.yaml" +server: + httpPortV4: 4000 + webSocket: + # Read buffer size in bytes (default: 4096) + readBufferSize: 4096 + # Write buffer size in bytes (default: 4096) + writeBufferSize: 4096 + # Maximum incoming message size in bytes (default: 1048576 = 1MB) + maxMessageSize: 1048576 + # Ping interval for keepalive (default: 30s). A client that sends neither + # a pong nor a request within twice this interval is disconnected. + pingInterval: 30s + # Maximum active subscriptions per WebSocket connection (default: 100) + maxSubscriptionsPerConnection: 100 + # Maximum requests a connection runs at once (default: 250) + maxConcurrentRequestsPerConnection: 250 + # Notifications queued per subscription for a slow reader (default: 256) + subscriptionBufferSize: 256 +``` + +The upgrade request must name a network (`///`, or a path that [domain aliasing](/operation/url) resolves to one); otherwise it is rejected with `400`. On shutdown eRPC closes every client connection with `1001` (going away) straight away so clients reconnect to another instance; in-flight requests get up to `server.waitBeforeShutdown` to answer first. + +## Indexer configuration + +`eth_subscribe` fan-out and deduplication are handled by an internal event indexer. Most deployments can leave this unset: + + + +```yaml filename="erpc.yaml" +indexer: + # Per-filter seen-set used to drop duplicate log / pending-tx + # notifications delivered by sibling upstreams. 0 = default (8192). + dedupWindowSize: 8192 +``` + + +```ts filename="erpc.ts" +import { createConfig } from "@erpc-cloud/config"; + +export default createConfig({ + indexer: { + dedupWindowSize: 8192, + }, +}); +``` + + + + + `indexer.canonicalChainDepth` is deprecated and ignored: reorged-out logs now come from the upstream's own `removed: true` notifications. + + +## Subscription compatibility knobs + +### Stripping `fromBlock: "0x0"` from `eth_subscribe` logs filters + +Some JSON-RPC clients include `fromBlock: "0x0"` in the `eth_subscribe("logs", {…})` filter as a "from genesis" marker. `eth_subscribe` is a live-stream RPC — `fromBlock` has no standardised meaning there — and on backends that prune historical data the subscribe call fails outright. eRPC can strip the field per-network so the live stream succeeds; historical logs stay retrievable via `eth_getLogs` in the normal way. + +```yaml filename="erpc.yaml" +projects: + - id: main + networks: + - architecture: evm + evm: + chainId: 1 + # Drops fromBlock: "0x0" / "0" from eth_subscribe logs filters + # before forwarding to upstream. Non-zero fromBlocks pass through. + stripSubscribeFromBlockZero: true +``` + +Only a string `fromBlock` that parses to zero (`"0x0"`, `"0"`, `"0x00"`, …) is stripped — any other value passes through unchanged. Leave unset for chains where upstreams handle the field correctly. The field can also be set once under `networkDefaults.evm`. + +### HTTP / WebSocket tip-sync considerations for strict consumers + +Some RPC consumers compare the block number returned by HTTP `eth_getBlockByNumber("latest")` against a tip they've observed via a WebSocket `newHeads` subscription and flag any lag as a health-check failure. On fast-block chains (sub-second to a few seconds) the HTTP round-trip can trivially exceed the inter-block interval, so the HTTP response carries a block that was current when it was generated but older than a `newHeads` notification that arrived in parallel. That behaviour is inherent to HTTP-vs-push timing and is visible with or without a proxy in front. + +eRPC minimises the gap in three ways: + +- **Per-block cache keying for moving tags.** The realtime cache entry for a `latest` / `finalized` request is keyed by the block number the tag currently resolves to, so each tip advance is a distinct key rather than pinning every call within the TTL to the same cached payload. +- **`enforceHighestBlock` response rewrite.** When the upstream returns a block number below the network's tracked tip, eRPC's [integrity check](/config/failsafe/integrity) fetches the specific higher block from another upstream and returns that instead (falling back to the highest valid block it has if none can serve it). +- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. Only heads from upstreams that are eligible to serve the network feed the floor, and it is bounded: a value no live upstream comes close to (a stale or poisoned entry) is ignored rather than trusted, so it cannot pin the served tip. It never lifts `latest` past [`servedTip.guaranteedFor`](/config/projects/networks) / `guaranteedMethods`, and it does not apply to requests scoped with `use-upstream`. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. + +Even so, a transient single-block lag can reach a client when the chain produces a block faster than an in-flight HTTP round-trip completes. If your consumer treats single-block HTTP-vs-WS lag as a hard failure, widen its tolerance to match the chain's block time rather than expect eRPC (or any proxy) to eliminate it. + +## Directives + +[Request directives](/operation/directives) can be set via headers or query parameters on the initial WebSocket upgrade request. They apply to every regular RPC request on that connection, batches included, subject to the same `allowClientDirectives` rules as HTTP: + +```javascript +// Pass directives via query params on the WebSocket URL +const ws = new WebSocket("ws://localhost:4000/main/evm/1?useUpstream=my-preferred-upstream"); +``` + +Subscriptions (`eth_subscribe`) ignore the `useUpstream` directive — they're delivered by the network's WebSocket upstreams as described above, so notifications keep flowing if any upstream slows down. diff --git a/docs/pages/reference/metrics.mdx b/docs/pages/reference/metrics.mdx index ce28c4623..b6e97c800 100644 --- a/docs/pages/reference/metrics.mdx +++ b/docs/pages/reference/metrics.mdx @@ -311,6 +311,9 @@ All metric names carry the `erpc_` prefix. Full definitions: Date: Mon, 28 Sep 2026 14:23:17 +0100 Subject: [PATCH 09/21] fix(network): feed the delivered-head floor from every head source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clients receive each head from whichever WebSocket upstream delivers it first, eligible to the selection policy or not. Feeding the floor only from eligible upstreams let HTTP "latest" sit a block behind a head a subscriber had just received whenever the fastest upstream was excluded — which a lag-based policy does routinely. Any known source whose poller accepted the head now feeds it; the floor stays bounded against live heads. Co-Authored-By: Claude Opus 5.5 --- docs/pages/operation/websocket.mdx | 2 +- erpc/networks_ws_tip_test.go | 20 ++++++++------------ erpc/subscription_manager.go | 13 +++++-------- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/docs/pages/operation/websocket.mdx b/docs/pages/operation/websocket.mdx index d48d1e978..3408a61b4 100644 --- a/docs/pages/operation/websocket.mdx +++ b/docs/pages/operation/websocket.mdx @@ -329,7 +329,7 @@ eRPC minimises the gap in three ways: - **Per-block cache keying for moving tags.** The realtime cache entry for a `latest` / `finalized` request is keyed by the block number the tag currently resolves to, so each tip advance is a distinct key rather than pinning every call within the TTL to the same cached payload. - **`enforceHighestBlock` response rewrite.** When the upstream returns a block number below the network's tracked tip, eRPC's [integrity check](/config/failsafe/integrity) fetches the specific higher block from another upstream and returns that instead (falling back to the highest valid block it has if none can serve it). -- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. Only heads from upstreams that are eligible to serve the network feed the floor, and it is bounded: a value no live upstream comes close to (a stale or poisoned entry) is ignored rather than trusted, so it cannot pin the served tip. It never lifts `latest` past [`servedTip.guaranteedFor`](/config/projects/networks) / `guaranteedMethods`, and it does not apply to requests scoped with `use-upstream`. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. +- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. Every head a subscriber can receive feeds the floor once the delivering upstream's tip tracker has accepted it, and the floor is bounded: a value no live upstream comes close to (a stale or poisoned entry) is ignored rather than trusted, so it cannot pin the served tip. It never lifts `latest` past [`servedTip.guaranteedFor`](/config/projects/networks) / `guaranteedMethods`, and it does not apply to requests scoped with `use-upstream`. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. Even so, a transient single-block lag can reach a client when the chain produces a block faster than an in-flight HTTP round-trip completes. If your consumer treats single-block HTTP-vs-WS lag as a hard failure, widen its tolerance to match the chain's block time rather than expect eRPC (or any proxy) to eliminate it. diff --git a/erpc/networks_ws_tip_test.go b/erpc/networks_ws_tip_test.go index 215da21d6..d86d83723 100644 --- a/erpc/networks_ws_tip_test.go +++ b/erpc/networks_ws_tip_test.go @@ -123,10 +123,9 @@ func TestNetworkHandle_SuggestLatestBlock_AdvancesNetworkTipBeforeFanOut(t *test "process-local high-water mark must cover the delivered WS tip") } -// Only a head from an upstream the selection policy keeps eligible lifts -// "latest": a fallback-tier (cordoned) or unknown source still feeds its own -// poller, but must not advertise a block no eligible upstream reports. -func TestNetworkHandle_SuggestLatestBlock_OnlyTipCandidatesLiftLatest(t *testing.T) { +// Clients receive heads from whichever source delivers first, so a head from +// a cordoned fallback lifts "latest" too; an unknown source does not. +func TestNetworkHandle_SuggestLatestBlock_EveryKnownSourceLiftsLatest(t *testing.T) { defer util.ResetGock() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -145,17 +144,14 @@ func TestNetworkHandle_SuggestLatestBlock_OnlyTipCandidatesLiftLatest(t *testing require.NotNil(t, fallback) handle := &networkHandle{nw: network} - handle.SuggestLatestBlock("ws:fallback-1", 1010) handle.SuggestLatestBlock("ws:unknown", 1020) - - assert.Equal(t, int64(1010), fallback.EvmStatePoller().LatestBlock(), - "the fallback's own poller still advances") assert.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx), - "a cordoned or unknown source must not lift latest") + "an unknown source must not lift latest") - handle.SuggestLatestBlock("ws:primary-1", 1001) - assert.Equal(t, int64(1001), network.EvmHighestLatestBlockNumber(ctx), - "an eligible upstream's head floors latest") + handle.SuggestLatestBlock("ws:fallback-1", 1010) + assert.Equal(t, int64(1010), fallback.EvmStatePoller().LatestBlock()) + assert.Equal(t, int64(1010), network.EvmHighestLatestBlockNumber(ctx), + "a head a client can receive from the cordoned fallback floors latest") } // The delivered-head floor is network-wide: a use-upstream-scoped request is diff --git a/erpc/subscription_manager.go b/erpc/subscription_manager.go index be7e70c84..b4e0ac8d2 100644 --- a/erpc/subscription_manager.go +++ b/erpc/subscription_manager.go @@ -440,14 +440,11 @@ func (h *networkHandle) SuggestLatestBlock(sourceId string, blockNumber int64) { return } poller.SuggestLatestBlock(blockNumber) - if poller.LatestBlock() < blockNumber { - return - } - for _, c := range h.nw.tipCandidateUpstreams(ctx, "*") { - if c.Id() == upstreamID { - h.nw.NoteObservedLatestBlock(h.nw.appCtx, blockNumber) - break - } + // Every source's heads reach clients (whichever delivers first), so + // every source feeds the floor — once its poller accepted the head, + // which excludes jumps still pending chain-id verification. + if poller.LatestBlock() >= blockNumber { + h.nw.NoteObservedLatestBlock(h.nw.appCtx, blockNumber) } return } From e16d0fe76c9ee65bb8ae1318bc7dbdf65694589c Mon Sep 17 00:00:00 2001 From: Jonny Date: Mon, 28 Sep 2026 14:23:17 +0100 Subject: [PATCH 10/21] fix(evm): don't exclude the stale responder from the tip re-fetch On fast chains the upstream that answered "latest" one block behind is often the one whose own feed just delivered the tip, and so the only upstream that already has it. Excluding it made the re-fetch return null from every other upstream and fail open to the stale block. A corrupted tip is still caught: every upstream returns null for it and pickHighestBlock keeps the served block. Co-Authored-By: Claude Opus 5.5 --- architecture/evm/eth_getBlockByNumber.go | 18 ++++++++---------- architecture/evm/eth_getBlockByNumber_test.go | 9 +++------ docs/pages/config/failsafe/integrity.mdx | 8 ++++---- docs/pages/reference/evm/method-handlers.mdx | 6 +++--- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/architecture/evm/eth_getBlockByNumber.go b/architecture/evm/eth_getBlockByNumber.go index d034b80a2..046acc258 100644 --- a/architecture/evm/eth_getBlockByNumber.go +++ b/architecture/evm/eth_getBlockByNumber.go @@ -182,7 +182,7 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common } } - nnr, err := forwardGetBlockByNumber(ctx, network, nq, nr, respBlockNumber, highestBlockNumber, itx) + nnr, err := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx) // This is needed in case highest block number is corrupted somehow and for example // it is requesting a very high non-existent block number. return pickHighestBlock(ctx, nnr, nr, err) @@ -210,7 +210,7 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common ).Inc() } } - nnr, err := forwardGetBlockByNumber(ctx, network, nq, nr, respBlockNumber, highestBlockNumber, itx) + nnr, err := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx) return pickHighestBlock(ctx, nnr, nr, err) default: return nr, re @@ -273,15 +273,13 @@ func enforceNonNullBlock(ctx context.Context, nq *common.NormalizedRequest, nr * ) } -// forwardGetBlockByNumber re-fetches blockNumber for original. Without a -// caller use-upstream selector, the stale responder is excluded since it -// likely lacks the block. +// forwardGetBlockByNumber re-fetches blockNumber for original, keeping the +// caller's directives (including any use-upstream selector) and skipping the +// cache. func forwardGetBlockByNumber( ctx context.Context, network common.Network, original *common.NormalizedRequest, - stale *common.NormalizedResponse, - staleBlockNumber int64, blockNumber int64, includeTx bool, ) (*common.NormalizedResponse, error) { @@ -293,11 +291,11 @@ func forwardGetBlockByNumber( return nil, err } newReq := common.NewNormalizedRequestFromJsonRpcRequest(request) + // The upstream that answered stale is not excluded: on fast chains it is + // often the one whose own feed just delivered the tip, i.e. the only one + // that already has the block. dr := original.Directives().Clone() dr.SkipCacheRead = "true" - if dr.UseUpstream == "" && staleBlockNumber > 0 && !stale.FromCache() { - dr.UseUpstream = fmt.Sprintf("!%s", stale.UpstreamId()) - } newReq.SetDirectives(dr) newReq.SetNetwork(network) newReq.CopyHttpContextFrom(original) diff --git a/architecture/evm/eth_getBlockByNumber_test.go b/architecture/evm/eth_getBlockByNumber_test.go index 17e6032ac..97dfe223c 100644 --- a/architecture/evm/eth_getBlockByNumber_test.go +++ b/architecture/evm/eth_getBlockByNumber_test.go @@ -313,12 +313,9 @@ func TestEnforceHighestBlock_LatestRefetchMissFailsOpen(t *testing.T) { require.NoError(t, err) assert.Equal(t, int64(99), bn) - // The stale responder is excluded; a cache hit has none to exclude. - want := "!rpc1" - if fromCache { - want = "" - } - assert.Equal(t, []string{want}, network.forwarded, "fromCache=%v", fromCache) + // A single re-fetch that keeps the request's (empty) selector: the + // stale responder may be the only upstream that has the tip block. + assert.Equal(t, []string{""}, network.forwarded, "fromCache=%v", fromCache) } } diff --git a/docs/pages/config/failsafe/integrity.mdx b/docs/pages/config/failsafe/integrity.mdx index acc0446b8..6f7373a0e 100644 --- a/docs/pages/config/failsafe/integrity.mdx +++ b/docs/pages/config/failsafe/integrity.mdx @@ -137,10 +137,10 @@ request-aware: a `use-upstream` selector scopes it to the targeted subset. `) unless the request already carries a `use-upstream` selector, which -is kept; `pickHighestBlock` keeps the higher of the two — protecting against a corrupted -state pointer. Cache hits are enforced the same way, so a cached block behind the tip is +eRPC re-requests that block (`SkipCacheRead=true`), keeping the request's own +`use-upstream` selector if any; the upstream that answered is not excluded, since on fast +chains it is often the only one that already has the tip block. `pickHighestBlock` keeps +the higher of the two — protecting against a corrupted state pointer. Cache hits are enforced the same way, so a cached block behind the tip is re-fetched rather than served. **Stale-tip cache write guard.** Under `enforceHighestBlock`, a realtime-finality diff --git a/docs/pages/reference/evm/method-handlers.mdx b/docs/pages/reference/evm/method-handlers.mdx index 65c5ac468..01f84e2e2 100644 --- a/docs/pages/reference/evm/method-handlers.mdx +++ b/docs/pages/reference/evm/method-handlers.mdx @@ -115,7 +115,7 @@ Resolution is lock-efficient: param values are extracted under a read lock, reso **eth_blockNumber highest-block enforcement.** The project pre-forward hook forwards the request normally, then compares the upstream's response against `EvmHighestLatestBlockNumber` from the network's block poller. If the poller head is higher, the response is replaced with a synthetic one containing the highest known block number and `erpc_upstream_stale_latest_block_total` is incremented. Cached responses are returned as-is without comparison: the TTL is the correctness control for stale cached data. Controlled by `networks[].evm.integrity.enforceHighestBlock` (default `true`). **eth_getBlockByNumber and eth_getBlockByHash.** Network.PostForward runs two checks: -- **`enforceHighestBlock`:** For `"latest"` or `"finalized"` tagged requests, if the response block number is behind the network's known highest, eRPC re-issues the request excluding the stale upstream, fetches the newer block, and returns whichever of the two is higher via `pickHighestBlock`. Increments `erpc_upstream_stale_latest_block_total` or `erpc_upstream_stale_finalized_block_total`. +- **`enforceHighestBlock`:** For `"latest"` or `"finalized"` tagged requests, if the response block number is behind the network's known highest, eRPC re-issues the request for that block number, fetches the newer block, and returns whichever of the two is higher via `pickHighestBlock`. The upstream that answered stale stays eligible for the re-fetch: on fast chains it is often the one whose own feed just delivered the tip. Increments `erpc_upstream_stale_latest_block_total` or `erpc_upstream_stale_finalized_block_total`. - **`enforceNonNullBlock`:** Converts null/empty results to `ErrEndpointMissingData`. For numeric block params this always applies. For tag params it only applies when `enforceNonNullTaggedBlocks` is true. Both branches are guarded by `emptyResultBeyondConfidence`. Upstream.PostForward runs opt-in block validation when directives are set: `ValidateTransactionsRoot`, `ValidateHeaderFieldLengths`, `ValidateTransactionFields`, and `ValidateTransactionBlockInfo`. Phantom/system transactions are handled specially — their blocks legitimately have `transactionsRoot = emptyTrieRoot` despite a non-empty transactions array. Recognized conventions: Polygon PoS / BSC (`from=0x0`, `gas=0x0`) and HyperEVM native/L1 system txs (synthetic signature `r=0x1` and `gasPrice=0`). @@ -295,7 +295,7 @@ upstreams: [ - `eth_call` and `eth_estimateGas` with one param always have `"latest"` appended before forwarding; callers never see this normalization. [] - `eth_chainId` synthetic response is skipped when `ShouldSkipCacheRead("")` is true, allowing force-refresh requests to reach real upstreams. [] - `eth_blockNumber` cached responses are returned as-is, bypassing the highest-block comparison. [] -- `eth_getBlockByNumber` `enforceHighestBlock` re-forward excludes the stale upstream only when the original response contained a real block number (non-zero). [] +- `eth_getBlockByNumber` `enforceHighestBlock` re-forward keeps every upstream eligible, including the one that answered stale, and skips the cache; a re-fetched block that no upstream has comes back empty and `pickHighestBlock` keeps the original block. [] - `emptyResultBeyondConfidence` with head=0 (unknown) fails open — any block is treated as potentially retryable. [] - `eth_sendRawTransaction` network-level probe uses `context.WithoutCancel` + 3-second fixed timeout; probe runs even when the parent context is expired. [] - `eth_sendRawTransaction` synthetic success requires hash cross-check: returned `hash` field must match the locally-derived hash; mismatch or missing field returns the original error. [] @@ -320,7 +320,7 @@ upstreams: [ 1. **eth_call / eth_estimateGas missing block param auto-injection** — a client sending only the call object (no block tag) gets `"latest"` silently appended. Source: . 2. **eth_chainId skip-cache-read bypasses all synthetic levels** — `x-erpc-skip-cache-read: true` causes all three hook levels to fall through to real upstreams. Source: . 3. **eth_blockNumber from-cache bypass** — cached responses skip the highest-block comparison; TTL is the correctness control for stale cached data. Source: . -4. **enforceHighestBlock excludes stale upstream only when block was extracted** — if the upstream returned a JSON-RPC error, the extracted block number is 0 and the upstream is NOT excluded from the re-forward. Source: . +4. **enforceHighestBlock never excludes the stale upstream** — the re-forward is a fresh request for the higher block number, eligible on every upstream; the stale-block counters increment only when the original response contained a real block number (non-zero). Source: , . 5. **pickHighestBlock safety** — always compares old vs new response rather than blindly returning the re-fetched block, guarding against corrupt poller state. If one response is null/empty, returns the other; if both are null/empty, returns the error from the re-forward rather than a null block. Source: . 6. **eth_getBlockReceipts excluded from default markEmptyAsErrorMethods** — empty arrays are correct for 0-tx blocks; inclusion causes retry storms. Source: . 7. **Phantom transactions (Polygon PoS / BSC / HyperEVM)** — blocks whose transactions are all system/phantom txs legitimately have `transactionsRoot = emptyTrieRoot` on a non-empty array; the integrity module's `transactionsRootConsistency` check skips the inconsistency error for these. Polygon PoS / BSC system txs use `from=0x0, gas=0x0`; HyperEVM native/L1 system txs instead carry a real-looking sender and non-zero gas and are marked by a synthetic signature `r=0x1` with zero `gasPrice` (which also covers the HYPE bridge case where `s=1` ≠ `from`). Source: . From 41facf08827931faaee760adf14ada0f95861fba Mon Sep 17 00:00:00 2001 From: Jonny Date: Tue, 6 Oct 2026 09:40:16 +0100 Subject: [PATCH 11/21] fix(clients): mark WebSocket reconnect jitter as non-cryptographic gosec G404 flags math/rand in the reconnect backoff. The jitter only spreads redials; annotate it like the repo's other jitter uses. Co-Authored-By: Claude Opus 5.5 --- clients/ws_json_rpc_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/ws_json_rpc_client.go b/clients/ws_json_rpc_client.go index d2f0de374..24a689e53 100644 --- a/clients/ws_json_rpc_client.go +++ b/clients/ws_json_rpc_client.go @@ -499,7 +499,7 @@ func (c *WsJsonRpcClient) readLoop() { if conn == nil { // Always wait before dialing so a peer that accepts and then // immediately drops the connection can't drive a hot loop. - wait := backoff/2 + rand.N(backoff/2+1) + wait := backoff/2 + rand.N(backoff/2+1) // #nosec G404 -- reconnect jitter, not security-sensitive backoff = min(time.Duration(float64(backoff)*wsReconnectFactor), wsReconnectMax) c.logger.Info().Dur("backoff", wait).Msg("attempting websocket reconnection") select { From d981c8b368e84b98f170b1d0f2b91ad2e4b24f52 Mon Sep 17 00:00:00 2001 From: Jonny Date: Tue, 6 Oct 2026 09:01:22 +0100 Subject: [PATCH 12/21] fix(network): don't let an all-missing hedge leg cancel its siblings The hedge keeper kept an ErrUpstreamsExhausted whose causes were all missing-data, on the premise that no sibling leg could do better. A sibling may be on an upstream this leg never tried, e.g. a leg the fallback escape sent to a slow fallback while another leg re-swept the primaries, so the fast miss cancelled the leg about to succeed. Such a result now keeps the race going; if every leg ends that way the hedge still returns the last one to the retry layer. Co-authored-by: Vitas Spokas <84668053+shpookas@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- erpc/network_executor.go | 15 --- erpc/networks_failover_escape_test.go | 13 +- erpc/networks_failover_hedge_test.go | 167 ++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 16 deletions(-) create mode 100644 erpc/networks_failover_hedge_test.go diff --git a/erpc/network_executor.go b/erpc/network_executor.go index 666dc70a2..5f6677a60 100644 --- a/erpc/network_executor.go +++ b/erpc/network_executor.go @@ -697,21 +697,6 @@ func (e *networkExecutor) runHedge( if uxe.Upstreams() == nil || len(uxe.Upstreams()) == 0 { return false } - // When every upstream returned -32004/missing-data, no sibling - // hedge leg can do better — they will all find the same upstreams - // consumed. Keep this result so the retry layer (shouldRetryWithReason) - // sees ErrUpstreamsExhausted directly and applies the 500ms delay. - causes := uxe.Errors() - allMissing := len(causes) > 0 - for _, c := range causes { - if !common.HasErrorCode(c, common.ErrCodeEndpointMissingData) { - allMissing = false - break - } - } - if allMissing { - return true - } } // Underlying-retryable wrapped errors (e.g. ErrUpstreamsExhausted // wrapping a 5xx) should continue racing for a healthier sibling. diff --git a/erpc/networks_failover_escape_test.go b/erpc/networks_failover_escape_test.go index 5f7f35357..bb828c178 100644 --- a/erpc/networks_failover_escape_test.go +++ b/erpc/networks_failover_escape_test.go @@ -169,6 +169,9 @@ func buildFailoverNetwork( networkConfig.Failover = &common.FailoverConfig{OnDefaultsExhausted: util.BoolPtr(true)} } networkConfig.Failsafe = opts.failsafe + if opts.network != nil { + opts.network(networkConfig) + } var policyEngine *policy.Engine if !opts.noPolicy { @@ -208,6 +211,10 @@ type failoverFixtureOpts struct { // mocks registers test-specific mocks ahead of the standard ones, before // any poller starts. mocks func() + // configure adjusts the upstream configs before the network is built. + configure func(cfgs []*common.UpstreamConfig) + // network adjusts the network config before the network is built. + network func(cfg *common.NetworkConfig) } func setupFailoverFixture( @@ -232,7 +239,11 @@ func setupFailoverFixture( mockEthCallReturning("rpc3.localhost", "0x3333") mockEthCallReturning("rpc4.localhost", "0x4444") - network, upr, mt := buildFailoverNetwork(t, ctx, failoverUpstreamConfigs(), opts) + cfgs := failoverUpstreamConfigs() + if opts.configure != nil { + opts.configure(cfgs) + } + network, upr, mt := buildFailoverNetwork(t, ctx, cfgs, opts) upsList := upr.GetNetworkUpstreams(ctx, util.EvmNetworkId(999)) require.Len(t, upsList, 4) diff --git a/erpc/networks_failover_hedge_test.go b/erpc/networks_failover_hedge_test.go new file mode 100644 index 000000000..cf6d1765f --- /dev/null +++ b/erpc/networks_failover_hedge_test.go @@ -0,0 +1,167 @@ +package erpc + +import ( + "context" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + promUtil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// unboundedAll drops the availability bound from every upstream, so each is +// called (and answers) for blocks above its polled head. +func unboundedAll(cfgs []*common.UpstreamConfig) { + for _, cfg := range cfgs { + cfg.Evm.BlockAvailability = nil + } +} + +// ethCallMock answers eth_call on host with body after delay. +func ethCallMock(host string, delay time.Duration, body string) { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return r.URL.Host == host && strings.Contains(util.SafeReadBody(r), "eth_call") + }). + Reply(200). + Delay(delay). + JSON([]byte(body)) +} + +// countEthCalls counts eth_call requests reaching host, ahead of its standard +// mock. +func countEthCalls(host string, hits *atomic.Int64) { + gock.New("http://" + host). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + // Filters run before host matching. + if r.URL.Host == host && strings.Contains(util.SafeReadBody(r), "eth_call") { + hits.Add(1) + } + return false + }). + Reply(200) +} + +const missingDataBody = `{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"header not found"}}` + +// hedgeFaster hedges well before a slow upstream answers. +func hedgeFaster() []*common.FailsafeConfig { + return []*common.FailsafeConfig{{ + MatchMethod: "*", + Hedge: &common.HedgePolicyConfig{Delay: common.NewStaticDuration(20 * time.Millisecond), MaxCount: 1}, + }} +} + +func forwardEthCall(t *testing.T, ctx context.Context, network *Network, id int, blockHex string) (string, error) { + t.Helper() + req := ethCallRequest(id, blockHex) + req.SetNetwork(network) + resp, err := network.Forward(ctx, req) + if err != nil { + return "", err + } + require.NotNil(t, resp) + defer resp.Release() + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + return strings.Trim(jrr.GetResultString(), `"`), nil +} + +// A hedge leg that finds every routed upstream missing the data must not +// cancel a sibling leg that escalated to the fallbacks and is still waiting. +func TestFailover_HedgeKeepsEscalatedSibling(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // The primaries miss, the escape sends one leg to the (slow) fallbacks, + // and the hedge leg misses on the primaries again. + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3e8", // 1000 + enableFailover: true, + configure: unboundedAll, + failsafe: hedgeFaster(), + mocks: func() { + ethCallMock("rpc1.localhost", 0, missingDataBody) + ethCallMock("rpc2.localhost", 0, missingDataBody) + ethCallMock("rpc3.localhost", 80*time.Millisecond, `{"jsonrpc":"2.0","id":1,"result":"0x3333"}`) + ethCallMock("rpc4.localhost", 80*time.Millisecond, `{"jsonrpc":"2.0","id":1,"result":"0x3333"}`) + }, + }) + + escape := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + escapeBefore := promUtil.ToFloat64(escape) + for i := 0; i < 5; i++ { + result, err := forwardEthCall(t, ctx, network, i, "0x3ea") + require.NoError(t, err, "iter %d", i) + assert.Equal(t, "0x3333", result, "iter %d", i) + } + assert.Equal(t, escapeBefore+5, promUtil.ToFloat64(escape)) +} + +// A fallback that already answered missing data must not let a hedge leg +// cancel another fallback that is still working on the request. +func TestFailover_HedgeKeepsSlowFallbackAfterFastFallbackMiss(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3e8", // 1000 + enableFailover: true, + configure: unboundedAll, + failsafe: hedgeFaster(), + mocks: func() { + ethCallMock("rpc1.localhost", 0, missingDataBody) + ethCallMock("rpc2.localhost", 0, missingDataBody) + ethCallMock("rpc3.localhost", 0, missingDataBody) + ethCallMock("rpc4.localhost", 80*time.Millisecond, `{"jsonrpc":"2.0","id":1,"result":"0x4444"}`) + }, + }) + + for i := 0; i < 5; i++ { + result, err := forwardEthCall(t, ctx, network, i, "0x3ea") + require.NoError(t, err, "iter %d", i) + assert.Equal(t, "0x4444", result, "iter %d", i) + } +} + +// A fallback whose polled head is ahead does not take reads the primaries +// serve: fallbacks are reached only through the escape. (The default +// policy's probeExcluded may still mirror the request to them in the +// background, so upstream hits are not counted here.) +func TestFailover_FallbackAheadDoesNotTakeServedReads(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3ea", // 1002 + enableFailover: true, + configure: unboundedAll, + }) + + escape := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues("main", "evm:999", "eth_call") + escapeBefore := promUtil.ToFloat64(escape) + for i := 0; i < 5; i++ { + result, err := forwardEthCall(t, ctx, network, i, "0x3ea") + require.NoError(t, err, "iter %d", i) + assert.Contains(t, []string{"0x1111", "0x2222"}, result, "iter %d", i) + } + assert.Equal(t, escapeBefore, promUtil.ToFloat64(escape)) +} From bd0fb54263883f7c31c2688c241a8cc36ccac21e Mon Sep 17 00:00:00 2001 From: Jonny Date: Tue, 6 Oct 2026 04:37:32 +0100 Subject: [PATCH 13/21] fix(ws): hold back fallback heads while a primary streams heads newHeads is subscribed on every WebSocket upstream and each head goes out from whichever source announces it first. With failover on, a fallback-tier upstream whose WebSocket is faster told clients about a block no primary had yet; their follow-up reads then missed on the primaries, or escaped to the fallback while the primaries were healthy. A fallback's head now reaches clients, and feeds the delivered-head floor, only while no upstream outside that tier which the selection policy routes to has a live newHeads subscription of its own. "Down" is the policy's verdict, as for reads, so custom policies that keep fallbacks in the ordered list behave the same. A network whose only WebSocket upstreams are fallbacks, or whose primaries' subscriptions are dead, keeps receiving fallback heads. A held-back head still updates that fallback's state poller. NetworkHandle.SuggestLatestBlock now reports whether the head may be delivered; the indexer drops it before dedup otherwise. Co-Authored-By: Claude Opus 5.5 --- docs/pages/operation/websocket.mdx | 4 +- erpc/networks.go | 9 +- erpc/networks_ws_tip_test.go | 99 +++++++++++++++++++ erpc/subscription_manager.go | 65 +++++++++--- indexer/adapters/wsupstream/adapter.go | 8 ++ .../wsupstream/adapter_reconnect_test.go | 6 +- indexer/indexer.go | 4 +- indexer/indexer_test.go | 34 ++++++- indexer/ingress.go | 9 +- indexer/integration_test.go | 3 +- 10 files changed, 210 insertions(+), 31 deletions(-) diff --git a/docs/pages/operation/websocket.mdx b/docs/pages/operation/websocket.mdx index 3408a61b4..6fd0aa9a4 100644 --- a/docs/pages/operation/websocket.mdx +++ b/docs/pages/operation/websocket.mdx @@ -137,7 +137,7 @@ ws.send(JSON.stringify({ ### Subscription features -- **Resilient delivery** — `newHeads` is subscribed on every connected WebSocket upstream of the network, and each block is delivered from whichever upstream announces it first. `logs` and `newPendingTransactions` are subscribed on every WebSocket upstream whose circuit breaker is closed; with [`failover.onDefaultsExhausted`](/config/projects/networks#failover) enabled, upstreams tagged `tier:fallback` are used only when every other upstream rejects the subscribe. +- **Resilient delivery** — `newHeads` is subscribed on every connected WebSocket upstream of the network, and each block is delivered from whichever upstream announces it first. With [`failover.onDefaultsExhausted`](/config/projects/networks#failover) enabled, heads from upstreams tagged `tier:fallback` are delivered only while no other upstream that the selection policy routes to has a live `newHeads` subscription, so clients are not told of a block only a fallback has while the primaries are up. A held-back head still updates that fallback's tip tracker. How quickly fallback heads take over when the primaries fail follows the selection policy's `evalInterval`. `logs` and `newPendingTransactions` are subscribed on every WebSocket upstream whose circuit breaker is closed; with [`failover.onDefaultsExhausted`](/config/projects/networks#failover) enabled, upstreams tagged `tier:fallback` are used only when every other upstream rejects the subscribe. - **No surprise disconnects** — upstream reconnects, scoring changes and brief outages never close your connection; eRPC re-subscribes upstream behind the scenes. eRPC closes a connection only on shutdown (`1001`), when the client stops answering pings, or when a `logs` subscription overflows its buffer (see below). - **Deduplication** — even though multiple upstreams are subscribed in parallel, you only receive each block, log or pending tx once. Logs are identified by `blockHash`, `transactionHash` and `logIndex`; a log missing any of them is passed through without deduplication. - **Shared subscriptions** — clients subscribing to the same event on the same network share one upstream subscription per upstream. Each client gets its own subscription ID and receives every notification independently. @@ -329,7 +329,7 @@ eRPC minimises the gap in three ways: - **Per-block cache keying for moving tags.** The realtime cache entry for a `latest` / `finalized` request is keyed by the block number the tag currently resolves to, so each tip advance is a distinct key rather than pinning every call within the TTL to the same cached payload. - **`enforceHighestBlock` response rewrite.** When the upstream returns a block number below the network's tracked tip, eRPC's [integrity check](/config/failsafe/integrity) fetches the specific higher block from another upstream and returns that instead (falling back to the highest valid block it has if none can serve it). -- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. Every head a subscriber can receive feeds the floor once the delivering upstream's tip tracker has accepted it, and the floor is bounded: a value no live upstream comes close to (a stale or poisoned entry) is ignored rather than trusted, so it cannot pin the served tip. It never lifts `latest` past [`servedTip.guaranteedFor`](/config/projects/networks) / `guaranteedMethods`, and it does not apply to requests scoped with `use-upstream`. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. +- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. Every head a subscriber can receive (so not a held-back fallback head) feeds the floor once the delivering upstream's tip tracker has accepted it, and the floor is bounded: a value no live upstream comes close to (a stale or poisoned entry) is ignored rather than trusted, so it cannot pin the served tip. It never lifts `latest` past [`servedTip.guaranteedFor`](/config/projects/networks) / `guaranteedMethods`, and it does not apply to requests scoped with `use-upstream`. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. Even so, a transient single-block lag can reach a client when the chain produces a block faster than an in-flight HTTP round-trip completes. If your consumer treats single-block HTTP-vs-WS lag as a hard failure, widen its tolerance to match the chain's block time rather than expect eRPC (or any proxy) to eliminate it. diff --git a/erpc/networks.go b/erpc/networks.go index bbd2a509d..9934a9300 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -3647,9 +3647,12 @@ func (n *Network) acquireRateLimitPermit(ctx context.Context, req *common.Normal // tierUpstreamsByGroup moves fallback-tier upstreams behind the rest, // preserving order within each tier. func tierUpstreamsByGroup(ups []common.Upstream) []common.Upstream { - return stablePartition(ups, func(u common.Upstream) bool { - return u.Config() != nil && u.Config().HasTag(common.TagTierFallback) - }) + return stablePartition(ups, isFallbackTier) +} + +// isFallbackTier reports whether u is tagged tier:fallback. +func isFallbackTier(u common.Upstream) bool { + return u.Config() != nil && u.Config().HasTag(common.TagTierFallback) } // partitionUpstreamsByLatestBlock moves upstreams whose polled head is known diff --git a/erpc/networks_ws_tip_test.go b/erpc/networks_ws_tip_test.go index d86d83723..bd3c69cf1 100644 --- a/erpc/networks_ws_tip_test.go +++ b/erpc/networks_ws_tip_test.go @@ -10,6 +10,7 @@ import ( "github.com/erpc/erpc/common" "github.com/erpc/erpc/data" "github.com/erpc/erpc/health" + "github.com/erpc/erpc/internal/policy" "github.com/erpc/erpc/thirdparty" "github.com/erpc/erpc/upstream" "github.com/erpc/erpc/util" @@ -154,6 +155,104 @@ func TestNetworkHandle_SuggestLatestBlock_EveryKnownSourceLiftsLatest(t *testing "a head a client can receive from the cordoned fallback floors latest") } +type fakeHeadSource bool + +func (f fakeHeadSource) HeadsLive() bool { return bool(f) } + +// With failover on, a fallback's head reaches clients (and lifts latest) only +// when no primary the policy routes to is streaming heads itself. The +// fallback's own poller sees every head regardless. +func TestNetworkHandle_FallbackHeadsHeldWhilePrimariesStream(t *testing.T) { + setup := func(t *testing.T, ctx context.Context) (*Network, map[string]*upstream.Upstream) { + network, ups, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", // 1000 + fallbackLatest: "0x3e8", + enableFailover: true, + }) + require.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx)) + byId := make(map[string]*upstream.Upstream, len(ups)) + for _, u := range ups { + byId[u.Id()] = u + } + return network, byId + } + + t.Run("HeldWhileAPrimaryStreams", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + network, ups := setup(t, ctx) + + handle := &networkHandle{nw: network, heads: map[string]headSource{ + "primary-1": fakeHeadSource(false), + "primary-2": fakeHeadSource(true), + "fallback-1": fakeHeadSource(true), + }} + assert.False(t, handle.SuggestLatestBlock("ws:fallback-1", 1010)) + assert.Equal(t, int64(1010), ups["fallback-1"].EvmStatePoller().LatestBlock(), + "the fallback's poller still sees its head") + assert.Equal(t, int64(1000), network.EvmHighestLatestBlockNumber(ctx), + "a held head must not lift latest") + + assert.True(t, handle.SuggestLatestBlock("ws:primary-2", 1001)) + assert.Equal(t, int64(1001), network.EvmHighestLatestBlockNumber(ctx)) + }) + + t.Run("DeliveredWhenNoPrimaryStreams", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + network, _ := setup(t, ctx) + + // Primaries connected over HTTP only, or with no live subscription. + for _, heads := range []map[string]headSource{ + {"fallback-1": fakeHeadSource(true)}, + {"primary-1": fakeHeadSource(false), "fallback-1": fakeHeadSource(true)}, + } { + handle := &networkHandle{nw: network, heads: heads} + assert.True(t, handle.SuggestLatestBlock("ws:fallback-1", 1010)) + } + assert.Equal(t, int64(1010), network.EvmHighestLatestBlockNumber(ctx)) + }) + + t.Run("DeliveredWhenPolicyExcludesPrimaries", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + network, ups := setup(t, ctx) + + handle := &networkHandle{nw: network, heads: map[string]headSource{ + "primary-1": fakeHeadSource(true), + "primary-2": fakeHeadSource(true), + "fallback-1": fakeHeadSource(true), + }} + require.False(t, handle.SuggestLatestBlock("ws:fallback-1", 1010)) + + ups["primary-1"].Cordon("*", "test") + ups["primary-2"].Cordon("*", "test") + policy.TickForTest(network.policyEngine, network.networkId, "*") + + assert.True(t, handle.SuggestLatestBlock("ws:fallback-1", 1011)) + assert.Equal(t, int64(1011), network.EvmHighestLatestBlockNumber(ctx)) + }) + + t.Run("DeliveredWithFailoverOff", func(t *testing.T) { + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x3e8", + fallbackLatest: "0x3e8", + }) + + handle := &networkHandle{nw: network, heads: map[string]headSource{ + "primary-1": fakeHeadSource(true), + "fallback-1": fakeHeadSource(true), + }} + assert.True(t, handle.SuggestLatestBlock("ws:fallback-1", 1010)) + }) +} + // The delivered-head floor is network-wide: a use-upstream-scoped request is // answered from its group's own head. func TestDeliveredHeadFloor_SkipsSelectorScopedRequests(t *testing.T) { diff --git a/erpc/subscription_manager.go b/erpc/subscription_manager.go index b4e0ac8d2..251e185b8 100644 --- a/erpc/subscription_manager.go +++ b/erpc/subscription_manager.go @@ -259,14 +259,22 @@ func (sm *SubscriptionManager) bootstrapNetwork(ctx context.Context, nw *Network return common.NewErrNoWsUpstreamAvailable(networkID) } - sm.idx.RegisterNetwork(&networkHandle{nw: nw}) - sm.idx.RegisterNetworkSelector(networkID, &subIngressSelector{nw: nw}) var adapterOpts wsupstream.Options if cfg := nw.cfg; cfg != nil && cfg.Evm != nil && cfg.Evm.StripSubscribeFromBlockZero != nil { adapterOpts.StripSubscribeFromBlockZero = *cfg.Evm.StripSubscribeFromBlockZero } + adapters := make(map[string]*wsupstream.Adapter, len(wsUpstreams)) + heads := make(map[string]headSource, len(wsUpstreams)) for _, up := range wsUpstreams { - adapter := wsupstream.New(up, networkID, sm.logger, adapterOpts) + if adapter := wsupstream.New(up, networkID, sm.logger, adapterOpts); adapter != nil { + adapters[up.Id()] = adapter + heads[up.Id()] = adapter + } + } + sm.idx.RegisterNetwork(&networkHandle{nw: nw, heads: heads}) + sm.idx.RegisterNetworkSelector(networkID, &subIngressSelector{nw: nw}) + for _, up := range wsUpstreams { + adapter := adapters[up.Id()] if adapter == nil { continue } @@ -416,38 +424,65 @@ func (sm *SubscriptionManager) recordFailureMetrics( // networkHandle adapts *Network to indexer.NetworkHandle. type networkHandle struct { nw *Network + // heads are the network's WebSocket ingresses by upstream id; fixed once + // the network is registered. + heads map[string]headSource +} + +// headSource is an ingress that streams newHeads. +type headSource interface { + // HeadsLive reports whether its newHeads subscription is live. + HeadsLive() bool } func (h *networkHandle) Id() string { return h.nw.networkId } // SuggestLatestBlock passes a head from the ingress "ws:" to -// that upstream's state poller. Once the poller has accepted it (a major -// jump is verified asynchronously first), a head from a tip candidate also -// advances the network's delivered-head floor before clients see it; a -// fallback-tier or policy-excluded upstream cannot lift "latest". -func (h *networkHandle) SuggestLatestBlock(sourceId string, blockNumber int64) { +// that upstream's state poller and reports whether clients may receive it +// (see deliversHeadsFrom). Once the poller has accepted a deliverable head (a +// major jump is verified asynchronously first), it also advances the +// network's delivered-head floor before clients see it. +func (h *networkHandle) SuggestLatestBlock(sourceId string, blockNumber int64) bool { upstreamID, ok := strings.CutPrefix(sourceId, "ws:") if !ok { - return + return true } ctx := context.Background() for _, u := range h.nw.upstreamsRegistry.GetNetworkUpstreams(ctx, h.nw.networkId) { if u.Id() != upstreamID { continue } + deliver := h.deliversHeadsFrom(ctx, u) poller := u.EvmStatePoller() if poller == nil || poller.IsObjectNull() { - return + return deliver } poller.SuggestLatestBlock(blockNumber) - // Every source's heads reach clients (whichever delivers first), so - // every source feeds the floor — once its poller accepted the head, - // which excludes jumps still pending chain-id verification. - if poller.LatestBlock() >= blockNumber { + if deliver && poller.LatestBlock() >= blockNumber { h.nw.NoteObservedLatestBlock(h.nw.appCtx, blockNumber) } - return + return deliver + } + return true +} + +// deliversHeadsFrom reports whether clients may receive u's heads. With +// failover on, a fallback-tier upstream's heads are held back while some +// upstream outside that tier, which the selection policy routes to, has a +// live newHeads subscription of its own: clients are not told of a block +// only a fallback has while the primaries are up. Whether they are up is the +// policy's verdict, as for reads. Without such a primary (none eligible, or +// none streaming heads) the fallbacks' heads are delivered. +func (h *networkHandle) deliversHeadsFrom(ctx context.Context, u common.Upstream) bool { + if h.nw.cfg == nil || !h.nw.cfg.Failover.Enabled() || !isFallbackTier(u) { + return true + } + for _, c := range h.nw.tipCandidateUpstreams(ctx, "*") { + if s := h.heads[c.Id()]; s != nil && !isFallbackTier(c) && s.HeadsLive() { + return false + } } + return true } var ( diff --git a/indexer/adapters/wsupstream/adapter.go b/indexer/adapters/wsupstream/adapter.go index d0814ea9b..0f3caa545 100644 --- a/indexer/adapters/wsupstream/adapter.go +++ b/indexer/adapters/wsupstream/adapter.go @@ -116,6 +116,14 @@ func New(up *upstream.Upstream, networkID string, logger *zerolog.Logger, opts O func (a *Adapter) Name() string { return "ws:" + a.upstreamID } +// HeadsLive reports whether the newHeads subscription is live on the +// current connection. +func (a *Adapter) HeadsLive() bool { + a.subsMu.Lock() + defer a.subsMu.Unlock() + return a.heads.id != "" +} + // Start registers the connection hooks and subscribes newHeads in the // background, detached from ctx. func (a *Adapter) Start(_ context.Context, nw indexer.NetworkHandle, sink indexer.Sink) error { diff --git a/indexer/adapters/wsupstream/adapter_reconnect_test.go b/indexer/adapters/wsupstream/adapter_reconnect_test.go index 672e152c0..ec46a84c7 100644 --- a/indexer/adapters/wsupstream/adapter_reconnect_test.go +++ b/indexer/adapters/wsupstream/adapter_reconnect_test.go @@ -26,8 +26,8 @@ import ( type fakeNetworkHandle struct{} -func (fakeNetworkHandle) Id() string { return "evm:123" } -func (fakeNetworkHandle) SuggestLatestBlock(string, int64) {} +func (fakeNetworkHandle) Id() string { return "evm:123" } +func (fakeNetworkHandle) SuggestLatestBlock(string, int64) bool { return true } type fakeSink struct { events chan indexer.StreamEvent @@ -235,10 +235,12 @@ func TestAdapterResubscribesWithRetryAfterReconnect(t *testing.T) { return send(ctx, nq, bypass) } + require.False(t, a.HeadsLive(), "no heads before the subscribe succeeds") require.NoError(t, a.Start(context.Background(), fakeNetworkHandle{}, sink)) require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond, "adapter never subscribed newHeads despite the failures stopping after %d", failuresPerEpoch) + require.True(t, a.HeadsLive()) require.GreaterOrEqual(t, forwardCalls.Load(), int64(failuresPerEpoch+1), "expected the subscribe to be retried through failures") diff --git a/indexer/indexer.go b/indexer/indexer.go index e382a5d88..11a592f3e 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -303,7 +303,9 @@ func (i *Indexer) Ingest(ev StreamEvent) { // Before dedup, so every source's head counts even if another source // already delivered it. if ev.Kind == KindNewHead && !ev.Block.Zero() && ev.SourceId != "" { - ns.handle.SuggestLatestBlock(ev.SourceId, ev.Block.Number) + if !ns.handle.SuggestLatestBlock(ev.SourceId, ev.Block.Number) { + return + } } // The upstream's removed flag is trusted as-is. diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go index fcbe50949..3d0866c88 100644 --- a/indexer/indexer_test.go +++ b/indexer/indexer_test.go @@ -18,7 +18,8 @@ type fakeNetwork struct { id string mu sync.Mutex - suggestedBy map[string][]int64 // sourceId -> block nums seen + suggestedBy map[string][]int64 // sourceId -> block nums seen + held map[string]struct{} // sourceIds whose heads are not delivered } func newFakeNetwork(id string) *fakeNetwork { @@ -29,10 +30,12 @@ func newFakeNetwork(id string) *fakeNetwork { } func (n *fakeNetwork) Id() string { return n.id } -func (n *fakeNetwork) SuggestLatestBlock(sourceId string, block int64) { +func (n *fakeNetwork) SuggestLatestBlock(sourceId string, block int64) bool { n.mu.Lock() + defer n.mu.Unlock() n.suggestedBy[sourceId] = append(n.suggestedBy[sourceId], block) - n.mu.Unlock() + _, held := n.held[sourceId] + return !held } type fakeEgress struct { @@ -271,6 +274,31 @@ func TestIndexer_NewHead_StalerDroppedKeepsStatePollerFed(t *testing.T) { } } +// A head the network does not deliver is still observed, and does not +// advance the dedup marker past the heads the other sources deliver. +func TestIndexer_NewHead_HeldSourceObservedNotDelivered(t *testing.T) { + idx := newIndexer(t) + nw := newFakeNetwork("evm:1") + nw.held = map[string]struct{}{"ws:fb": {}} + idx.RegisterNetwork(nw) + eg := &fakeEgress{name: "eg1", acceptAllHeads: true} + idx.Attach(eg) + + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:fb", Block: BlockRef{Number: 101, Hash: "0xBBB"}}) + if got := eg.count(); got != 0 { + t.Fatalf("held head delivered: got %d", got) + } + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:up1", Block: BlockRef{Number: 101, Hash: "0xBBB"}}) + if got := eg.count(); got != 1 { + t.Fatalf("the delivering source's head must go out: want 1, got %d", got) + } + nw.mu.Lock() + defer nw.mu.Unlock() + if len(nw.suggestedBy["ws:fb"]) != 1 { + t.Fatalf("held source must still be observed, got %v", nw.suggestedBy) + } +} + func TestIndexer_Log_RefcountFanOutAndTeardown(t *testing.T) { idx := newIndexer(t) nw := newFakeNetwork("evm:1") diff --git a/indexer/ingress.go b/indexer/ingress.go index 3df2d4086..26338e672 100644 --- a/indexer/ingress.go +++ b/indexer/ingress.go @@ -11,10 +11,11 @@ type Sink interface { // NetworkHandle is the part of a network an ingress may touch. type NetworkHandle interface { Id() string - // SuggestLatestBlock reports a head observed by source. It is called - // before dedup and fan-out, so every source's observation counts and - // the network tip moves before clients see the head. - SuggestLatestBlock(sourceId string, blockNumber int64) + // SuggestLatestBlock reports a head observed by source and whether + // clients may receive it. It is called before dedup and fan-out, so + // every source's observation counts and the network tip moves before + // clients see the head; a head it does not deliver is dropped. + SuggestLatestBlock(sourceId string, blockNumber int64) (deliver bool) } // EventIngress turns a transport-specific subscription into StreamEvents diff --git a/indexer/integration_test.go b/indexer/integration_test.go index 976144854..ebecd78a8 100644 --- a/indexer/integration_test.go +++ b/indexer/integration_test.go @@ -66,13 +66,14 @@ type stubNetwork struct { } func (s *stubNetwork) Id() string { return s.id } -func (s *stubNetwork) SuggestLatestBlock(sourceID string, block int64) { +func (s *stubNetwork) SuggestLatestBlock(sourceID string, block int64) bool { s.mu.Lock() if s.suggestions == nil { s.suggestions = make(map[string][]int64) } s.suggestions[sourceID] = append(s.suggestions[sourceID], block) s.mu.Unlock() + return true } type recordingEgress struct { From 92f54805bf710278e3eb8a221c5fe48ceaead111 Mon Sep 17 00:00:00 2001 From: Jonny Date: Tue, 6 Oct 2026 09:01:10 +0100 Subject: [PATCH 14/21] fix(ws): keep filter subscriptions on the same tiers as heads logs and newPendingTransactions picked their upstreams once, when the first client subscribed: on every primary whose eth_subscribe breaker was closed, on the fallbacks only if every primary rejected the subscribe. Nothing revisited that for the filter's lifetime, so a filter went silent when its primaries dropped it, stayed on the fallbacks after the primaries recovered, and never reached a primary whose first subscribe failed. A filter now belongs on every primary the selection policy routes to, and on the fallback tier while none of those has it live; the indexer rechecks this on every head, adding and removing the filter as that changes. The fallbacks hand a filter back only once a primary has it live, so there is no gap. An upstream whose subscribe fails keeps the filter and retries it in the background, like its other subscriptions. Only heads a tip tracker has accepted reach subscribers: a major jump is verified before it is delivered, and a source with no tracker is withheld, so an unverified head can't advance the indexer's dedup marker. The upstream eth_unsubscribe outlives a caller that already gave up but keeps a live caller deadline. The indexer and the subscription manager key a network by project and network, so two projects that define the same network keep their own upstreams. A filter subscribe is bounded by the attempt timeout, since callers hold locks across it and a dropped reply on a busy connection would otherwise never time out. Co-Authored-By: Claude Opus 5.5 --- docs/pages/operation/websocket.mdx | 6 +- erpc/networks_ws_tip_test.go | 71 +++++++ erpc/subscription_manager.go | 102 +++++---- erpc/ws_server_hardening_test.go | 52 +++++ erpc/ws_server_test.go | 4 +- indexer/adapters/nullingress/adapter.go | 2 + indexer/adapters/wsupstream/adapter.go | 79 +++++-- .../wsupstream/adapter_reconnect_test.go | 201 +++++++++++++++++- indexer/indexer.go | 136 ++++++++++-- indexer/indexer_test.go | 121 +++++++++++ indexer/ingress.go | 9 +- 11 files changed, 695 insertions(+), 88 deletions(-) diff --git a/docs/pages/operation/websocket.mdx b/docs/pages/operation/websocket.mdx index 6fd0aa9a4..0bb3e01ad 100644 --- a/docs/pages/operation/websocket.mdx +++ b/docs/pages/operation/websocket.mdx @@ -60,7 +60,7 @@ ws.send(JSON.stringify([ // Response is a JSON array ``` -All eRPC features work identically for WebSocket RPC requests: authentication, method filters, caching, upstream scoring, failsafe policies (retry, hedge, circuit breaker), rate limiting, multiplexing (request deduplication), EVM integrity checks, and metrics. Each request is bounded by `server.maxTimeout`, and a connection runs at most `server.webSocket.maxConcurrentRequestsPerConnection` requests at once — further messages wait until one finishes. +All eRPC features work identically for WebSocket RPC requests: authentication, method filters, caching, upstream scoring, failsafe policies (retry, hedge, circuit breaker), rate limiting, multiplexing (request deduplication), EVM integrity checks, and metrics. Each request is bounded by `server.maxTimeout`, and a connection processes at most `server.webSocket.maxConcurrentRequestsPerConnection` messages at once — further messages wait until one finishes. A batch is one message; its requests run concurrently, as over HTTP. Credentials (`secret` query parameter or headers, JWT, SIWE, client IP) come from the upgrade request: an upgrade that no auth strategy accepts is refused with `401`, and each request on the connection is still authenticated, with method scoping and rate limits, as over HTTP. ## Subscriptions @@ -137,7 +137,7 @@ ws.send(JSON.stringify({ ### Subscription features -- **Resilient delivery** — `newHeads` is subscribed on every connected WebSocket upstream of the network, and each block is delivered from whichever upstream announces it first. With [`failover.onDefaultsExhausted`](/config/projects/networks#failover) enabled, heads from upstreams tagged `tier:fallback` are delivered only while no other upstream that the selection policy routes to has a live `newHeads` subscription, so clients are not told of a block only a fallback has while the primaries are up. A held-back head still updates that fallback's tip tracker. How quickly fallback heads take over when the primaries fail follows the selection policy's `evalInterval`. `logs` and `newPendingTransactions` are subscribed on every WebSocket upstream whose circuit breaker is closed; with [`failover.onDefaultsExhausted`](/config/projects/networks#failover) enabled, upstreams tagged `tier:fallback` are used only when every other upstream rejects the subscribe. +- **Resilient delivery** — `newHeads` is subscribed on every connected WebSocket upstream of the network, and each block is delivered from whichever upstream announces it first. With [`failover.onDefaultsExhausted`](/config/projects/networks#failover) enabled, heads from upstreams tagged `tier:fallback` are delivered only while no other upstream that the selection policy routes to has a live `newHeads` subscription, so clients are not told of a block only a fallback has while the primaries are up. A held-back head still updates that fallback's tip tracker. How quickly fallback heads take over when the primaries fail follows the selection policy's `evalInterval`. `logs` and `newPendingTransactions` are subscribed on every WebSocket upstream; with [`failover.onDefaultsExhausted`](/config/projects/networks#failover) enabled, on every one outside `tier:fallback` that the selection policy routes to, and on the `tier:fallback` upstreams only while none of those has the subscription live. This is rechecked on every head, so the fallbacks take over when the primaries drop a subscription (or the policy excludes them) and hand it back once a primary has it live again; a subscribe that fails is retried in the background. - **No surprise disconnects** — upstream reconnects, scoring changes and brief outages never close your connection; eRPC re-subscribes upstream behind the scenes. eRPC closes a connection only on shutdown (`1001`), when the client stops answering pings, or when a `logs` subscription overflows its buffer (see below). - **Deduplication** — even though multiple upstreams are subscribed in parallel, you only receive each block, log or pending tx once. Logs are identified by `blockHash`, `transactionHash` and `logIndex`; a log missing any of them is passed through without deduplication. - **Shared subscriptions** — clients subscribing to the same event on the same network share one upstream subscription per upstream. Each client gets its own subscription ID and receives every notification independently. @@ -329,7 +329,7 @@ eRPC minimises the gap in three ways: - **Per-block cache keying for moving tags.** The realtime cache entry for a `latest` / `finalized` request is keyed by the block number the tag currently resolves to, so each tip advance is a distinct key rather than pinning every call within the TTL to the same cached payload. - **`enforceHighestBlock` response rewrite.** When the upstream returns a block number below the network's tracked tip, eRPC's [integrity check](/config/failsafe/integrity) fetches the specific higher block from another upstream and returns that instead (falling back to the highest valid block it has if none can serve it). -- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. Every head a subscriber can receive (so not a held-back fallback head) feeds the floor once the delivering upstream's tip tracker has accepted it, and the floor is bounded: a value no live upstream comes close to (a stale or poisoned entry) is ignored rather than trusted, so it cannot pin the served tip. It never lifts `latest` past [`servedTip.guaranteedFor`](/config/projects/networks) / `guaranteedMethods`, and it does not apply to requests scoped with `use-upstream`. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. +- **Delivered-head floor.** When an upstream WebSocket delivers a `newHeads` notification, eRPC updates that upstream's tip tracker before fanning the head out to clients, and records it as a floor for the network's `latest`. A head reaches subscribers, and feeds the floor, only once the delivering upstream's tip tracker has accepted it; a head far ahead of the tracker is verified first and is not delivered, so subscribers resume with the next head. Held-back fallback heads are not delivered either. The floor is bounded: a value no live upstream comes close to (a stale or poisoned entry) is ignored rather than trusted, so it cannot pin the served tip. When `servedTip` is enabled for `latest`, it never lifts `latest` past [`servedTip.guaranteedFor`](/config/projects/networks) / `guaranteedMethods`. It does not apply to requests scoped with `use-upstream`. With shared state configured the floor is also propagated to other instances in the background, so they converge within the propagation delay rather than instantly. Even so, a transient single-block lag can reach a client when the chain produces a block faster than an in-flight HTTP round-trip completes. If your consumer treats single-block HTTP-vs-WS lag as a hard failure, widen its tolerance to match the chain's block time rather than expect eRPC (or any proxy) to eliminate it. diff --git a/erpc/networks_ws_tip_test.go b/erpc/networks_ws_tip_test.go index bd3c69cf1..fa72fdccb 100644 --- a/erpc/networks_ws_tip_test.go +++ b/erpc/networks_ws_tip_test.go @@ -300,3 +300,74 @@ func TestDeliveredHeadFloor_IsScopedPerProject(t *testing.T) { assert.Equal(t, int64(1001), network.latestBlockShared.GetValue()) assert.Equal(t, int64(0), sibling.latestBlockShared.GetValue()) } + +// Filter subscriptions follow the same tiers as heads: eligible primaries +// carry them, the fallback tier stands in, and without failover every +// WebSocket upstream is a default. +func TestTierWsIngresses(t *testing.T) { + ws := []common.Upstream{ + common.NewFakeUpstream("p1"), + common.NewFakeUpstream("p2"), + common.NewFakeUpstream("fb", common.WithTags(common.TagTierFallback)), + } + eligible := map[string]struct{}{"p1": {}, "fb": {}} + + d, f := tierWsIngresses(ws, eligible, true) + assert.Equal(t, []string{"ws:p1"}, d, "an ineligible primary does not carry filters") + assert.Equal(t, []string{"ws:fb"}, f) + + d, f = tierWsIngresses(ws, nil, true) + assert.Empty(t, d, "no eligible primary: only the fallbacks remain") + assert.Equal(t, []string{"ws:fb"}, f) + + d, f = tierWsIngresses(ws, nil, false) + assert.Equal(t, []string{"ws:p1", "ws:p2", "ws:fb"}, d) + assert.Empty(t, f) +} + +// A head far past the poller's (a major jump) is verified asynchronously +// before the poller takes it. Until then it must not reach clients: once +// delivered it would also advance the indexer's head dedup marker, and a +// head from an endpoint now answering for another chain would suppress +// every real head below it. +func TestNetworkHandle_SuggestLatestBlock_WithholdsUnacceptedMajorJump(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, u := setupWsTipNetwork(t, ctx, "rpc2", "http://rpc2.localhost") + u.EvmStatePoller().SuggestLatestBlock(1000) + require.Equal(t, int64(1000), u.EvmStatePoller().LatestBlock()) + + // From here the endpoint answers for another chain. + util.ResetGock() + gock.New("http://rpc2.localhost").Post("").Persist(). + Filter(func(r *http.Request) bool { return strings.Contains(util.SafeReadBody(r), `eth_chainId`) }). + Reply(200).JSON([]byte(`{"result":"0x1"}`)) + + handle := &networkHandle{nw: network} + assert.False(t, handle.SuggestLatestBlock("ws:rpc2", 5_000_000), "an unaccepted major jump must not reach clients") + time.Sleep(300 * time.Millisecond) // let the async verification finish (and fail) + assert.Equal(t, int64(1000), u.EvmStatePoller().LatestBlock()) + assert.Less(t, network.deliveredLatestBlock.Load(), int64(5_000_000)) + assert.True(t, handle.SuggestLatestBlock("ws:rpc2", 1001)) + assert.True(t, handle.SuggestLatestBlock("ws:rpc2", 1002)) +} + +// Heads are delivered only once a tip tracker has accepted them, so a head +// from a source the network can't verify (no such upstream, or one with no +// tracker) is withheld and leaves the delivered-head floor alone. +func TestNetworkHandle_SuggestLatestBlock_WithholdsUnverifiableSource(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _ := setupWsTipNetwork(t, ctx, "rpc3", "http://rpc3.localhost") + handle := &networkHandle{nw: network} + floor := network.deliveredLatestBlock.Load() + + assert.False(t, handle.SuggestLatestBlock("ws:not-in-this-network", 5_000_000)) + assert.Equal(t, floor, network.deliveredLatestBlock.Load()) +} diff --git a/erpc/subscription_manager.go b/erpc/subscription_manager.go index 251e185b8..23927a68f 100644 --- a/erpc/subscription_manager.go +++ b/erpc/subscription_manager.go @@ -13,7 +13,6 @@ import ( "github.com/erpc/erpc/indexer/adapters/wsclient" "github.com/erpc/erpc/indexer/adapters/wsupstream" "github.com/erpc/erpc/telemetry" - "github.com/erpc/erpc/upstream" "github.com/rs/zerolog" ) @@ -38,7 +37,7 @@ type SubscriptionManager struct { connMu sync.Mutex conns map[string]*connEntry // connId -> entry - networks sync.Map // networkId -> struct{}, once bootstrapped + networks sync.Map // indexerNetworkKey -> struct{}, once bootstrapped bootstrapMu sync.Mutex } @@ -123,14 +122,15 @@ func (sm *SubscriptionManager) Subscribe( return nil, err } - kind, filterHash, err := sm.resolveSubscription(ctx, networkId, subType, jrReq.Params) + key := indexerNetworkKey(nw) + kind, filterHash, err := sm.resolveSubscription(ctx, key, subType, jrReq.Params) if err != nil { sm.recordFailureMetrics(project, nw, method, reqFinality, start, nq, err) return nil, err } - if err := conn.adapter.AddSubscription(clientSubID, networkId, kind, filterHash, maxSubs); err != nil { - sm.releaseFilter(ctx, networkId, kind, filterHash) + if err := conn.adapter.AddSubscription(clientSubID, key, kind, filterHash, maxSubs); err != nil { + sm.releaseFilter(ctx, key, kind, filterHash) if errors.Is(err, wsclient.ErrLimitExceeded) { err = common.NewErrSubscriptionLimitExceeded(maxSubs) } @@ -245,12 +245,13 @@ func (sm *SubscriptionManager) releaseFilter(ctx context.Context, networkID stri // per WebSocket upstream, once. func (sm *SubscriptionManager) bootstrapNetwork(ctx context.Context, nw *Network) error { networkID := nw.networkId - if _, ok := sm.networks.Load(networkID); ok { + key := indexerNetworkKey(nw) + if _, ok := sm.networks.Load(key); ok { return nil } sm.bootstrapMu.Lock() defer sm.bootstrapMu.Unlock() - if _, ok := sm.networks.Load(networkID); ok { + if _, ok := sm.networks.Load(key); ok { return nil } @@ -266,57 +267,69 @@ func (sm *SubscriptionManager) bootstrapNetwork(ctx context.Context, nw *Network adapters := make(map[string]*wsupstream.Adapter, len(wsUpstreams)) heads := make(map[string]headSource, len(wsUpstreams)) for _, up := range wsUpstreams { - if adapter := wsupstream.New(up, networkID, sm.logger, adapterOpts); adapter != nil { + if adapter := wsupstream.New(up, key, sm.logger, adapterOpts); adapter != nil { adapters[up.Id()] = adapter heads[up.Id()] = adapter } } sm.idx.RegisterNetwork(&networkHandle{nw: nw, heads: heads}) - sm.idx.RegisterNetworkSelector(networkID, &subIngressSelector{nw: nw}) + sm.idx.RegisterNetworkSelector(key, &subIngressSelector{nw: nw}) for _, up := range wsUpstreams { adapter := adapters[up.Id()] if adapter == nil { continue } - if err := sm.idx.AddIngress(ctx, networkID, adapter); err != nil { + if err := sm.idx.AddIngress(ctx, key, adapter); err != nil { sm.logger.Warn().Err(err).Str("upstreamId", up.Id()). Msg("failed to register upstream ingress with indexer") } } - sm.networks.Store(networkID, struct{}{}) + sm.networks.Store(key, struct{}{}) return nil } -// subIngressSelector picks the ingresses of a filter subscribe the way the -// HTTP path picks upstreams: by score for eth_subscribe, skipping upstreams -// whose circuit breaker is open. Fallback-tier upstreams form the fallback -// tier only when the network's failover is enabled, as over HTTP. +// subIngressSelector picks the ingresses that carry a filter subscription. +// With failover on, the defaults are the upstreams outside the fallback tier +// that the selection policy routes to, and the fallback-tier upstreams stand +// in for them (see indexer.IngressSelector): the same rule newHeads follows +// (see networkHandle.deliversHeadsFrom). Without failover every WebSocket +// upstream is a default. type subIngressSelector struct { nw *Network } func (s *subIngressSelector) Select(_, _ string, _ []interface{}) (defaults, fallbacks []string) { - ups, err := s.nw.upstreamsRegistry.GetSortedUpstreams(context.Background(), s.nw.networkId, MethodEthSubscribe) - if err != nil { - return nil, nil - } - + ctx := context.Background() failoverOn := s.nw.cfg != nil && s.nw.cfg.Failover.Enabled() - for _, u := range ups { - up, ok := u.(*upstream.Upstream) - if !ok { - continue - } - cfg := up.Config() - if cfg == nil || !upstream.IsWsEndpoint(cfg.Endpoint) || up.IsDown(MethodEthSubscribe) { - continue + eligible := make(map[string]struct{}) + if failoverOn { + for _, u := range s.nw.tipCandidateUpstreams(ctx, "*") { + eligible[u.Id()] = struct{}{} } + } + var ws []common.Upstream + for _, up := range s.nw.upstreamsRegistry.GetWsUpstreams(ctx, s.nw.networkId) { + ws = append(ws, up) + } + return tierWsIngresses(ws, eligible, failoverOn) +} + +// tierWsIngresses names the ingresses of the WebSocket upstreams ws: with +// failover on, the eligible ones outside the fallback tier are the defaults +// and the fallback tier the fallbacks; otherwise all are defaults. +func tierWsIngresses(ws []common.Upstream, eligible map[string]struct{}, failoverOn bool) (defaults, fallbacks []string) { + for _, up := range ws { name := "ws:" + up.Id() - if failoverOn && cfg.HasTag(common.TagTierFallback) { + switch { + case !failoverOn: + defaults = append(defaults, name) + case isFallbackTier(up): fallbacks = append(fallbacks, name) - continue + default: + if _, ok := eligible[up.Id()]; ok { + defaults = append(defaults, name) + } } - defaults = append(defaults, name) } return defaults, fallbacks } @@ -435,12 +448,20 @@ type headSource interface { HeadsLive() bool } -func (h *networkHandle) Id() string { return h.nw.networkId } +func (h *networkHandle) Id() string { return indexerNetworkKey(h.nw) } + +// indexerNetworkKey names nw in the indexer and its adapters. Projects can +// define the same network, each with its own upstreams, so the key includes +// the project. +func indexerNetworkKey(nw *Network) string { return nw.projectId + "/" + nw.networkId } // SuggestLatestBlock passes a head from the ingress "ws:" to -// that upstream's state poller and reports whether clients may receive it -// (see deliversHeadsFrom). Once the poller has accepted a deliverable head (a -// major jump is verified asynchronously first), it also advances the +// that upstream's state poller and reports whether clients may receive it: +// only once the poller has accepted it (a major jump is verified +// asynchronously first, so that head is not delivered) and the fallback-tier +// gate allows it (see deliversHeadsFrom). A head from an upstream with no +// poller (not bootstrapped, e.g. its chain could not be verified) or one the +// network doesn't know is not delivered. A delivered head also advances the // network's delivered-head floor before clients see it. func (h *networkHandle) SuggestLatestBlock(sourceId string, blockNumber int64) bool { upstreamID, ok := strings.CutPrefix(sourceId, "ws:") @@ -455,15 +476,16 @@ func (h *networkHandle) SuggestLatestBlock(sourceId string, blockNumber int64) b deliver := h.deliversHeadsFrom(ctx, u) poller := u.EvmStatePoller() if poller == nil || poller.IsObjectNull() { - return deliver + return false } poller.SuggestLatestBlock(blockNumber) - if deliver && poller.LatestBlock() >= blockNumber { - h.nw.NoteObservedLatestBlock(h.nw.appCtx, blockNumber) + if !deliver || poller.LatestBlock() < blockNumber { + return false } - return deliver + h.nw.NoteObservedLatestBlock(h.nw.appCtx, blockNumber) + return true } - return true + return false } // deliversHeadsFrom reports whether clients may receive u's heads. With diff --git a/erpc/ws_server_hardening_test.go b/erpc/ws_server_hardening_test.go index 86b9e9192..e5e6e6b88 100644 --- a/erpc/ws_server_hardening_test.go +++ b/erpc/ws_server_hardening_test.go @@ -7,6 +7,7 @@ import ( "io" "net" "net/http" + "net/http/httptest" "strings" "sync/atomic" "testing" @@ -536,3 +537,54 @@ func TestWebSocket_NotificationsGetNoReply(t *testing.T) { _, msg, err := conn.ReadMessage() require.Error(t, err, "notifications must not be answered, got %s", msg) } + +// Two projects can define the same network, each with its own upstreams. A +// subscription on the second project must be served by that project's +// upstreams, not by the first project's network that was bootstrapped first. +func TestWebSocket_ProjectsSharingANetworkUseTheirOwnUpstreams(t *testing.T) { + setupGock() + defer util.ResetGock() + + var subscribesA, subscribesB atomic.Int32 + mockWith := func(count *atomic.Int32, subID string) *httptest.Server { + return mockWsUpstream(t, func(conn *websocket.Conn) { + standardMockWsHandler(conn, func(method string, id interface{}, _ map[string]interface{}) { + result := interface{}("0x1") + if method == "eth_subscribe" { + count.Add(1) + result = subID + } + mockWriteJSON(conn, map[string]interface{}{"jsonrpc": "2.0", "id": id, "result": result}) + }) + }) + } + upA, upB := mockWith(&subscribesA, "0xa"), mockWith(&subscribesB, "0xb") + defer upA.Close() + defer upB.Close() + + cfg := standardWsConfig("ws" + strings.TrimPrefix(upA.URL, "http")) + second := *cfg.Projects[0] + second.Id = "test_ws2" + second.Upstreams = []*common.UpstreamConfig{ + {Id: "http-upstream", Type: common.UpstreamTypeEvm, Endpoint: "http://rpc1.localhost", Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + {Id: "ws-upstream", Type: common.UpstreamTypeEvm, Endpoint: "ws" + strings.TrimPrefix(upB.URL, "http"), Evm: &common.EvmUpstreamConfig{ChainId: 123}}, + } + cfg.Projects = append(cfg.Projects, &second) + addr, cleanup := setupTestERPCServer(t, cfg) + defer cleanup() + + subscribe := func(project string) { + t.Helper() + conn, _, err := websocket.DefaultDialer.Dial(fmt.Sprintf("ws://%s/%s/evm/123", addr, project), nil) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + resp := sendAndReceive(t, conn, `{"jsonrpc":"2.0","id":1,"method":"eth_subscribe","params":["newHeads"]}`) + require.Nil(t, resp["error"], "subscribe on %s: %v", project, resp) + } + + subscribe("test_ws") + require.Eventually(t, func() bool { return subscribesA.Load() > 0 }, 5*time.Second, 20*time.Millisecond) + subscribe("test_ws2") + assert.Eventually(t, func() bool { return subscribesB.Load() > 0 }, 5*time.Second, 20*time.Millisecond, + "the second project's subscription must subscribe on its own upstream") +} diff --git a/erpc/ws_server_test.go b/erpc/ws_server_test.go index 919ac6cbb..1fc430f7c 100644 --- a/erpc/ws_server_test.go +++ b/erpc/ws_server_test.go @@ -963,7 +963,9 @@ func TestWebSocket_SubscriptionDedup(t *testing.T) { "method": "eth_subscription", "params": map[string]interface{}{ "subscription": upstreamSubId, - "result": map[string]interface{}{"number": "0x999"}, + // One block past the mocked poller head (256); a + // major jump would be held for verification. + "result": map[string]interface{}{"number": "0x101"}, }, }) }() diff --git a/indexer/adapters/nullingress/adapter.go b/indexer/adapters/nullingress/adapter.go index 475dfbe84..77ccbfd48 100644 --- a/indexer/adapters/nullingress/adapter.go +++ b/indexer/adapters/nullingress/adapter.go @@ -60,6 +60,8 @@ func (a *Adapter) RemoveFilter(context.Context, string, string) error { return nil } +func (a *Adapter) FilterLive(string, string) bool { return true } + // Push injects an event. It blocks when the buffer is full, so a stalled // indexer shows up as a test timeout rather than silent drops. func (a *Adapter) Push(ev indexer.StreamEvent) { diff --git a/indexer/adapters/wsupstream/adapter.go b/indexer/adapters/wsupstream/adapter.go index 0f3caa545..d1abfdd96 100644 --- a/indexer/adapters/wsupstream/adapter.go +++ b/indexer/adapters/wsupstream/adapter.go @@ -50,6 +50,8 @@ type Adapter struct { retryMin time.Duration retryMax time.Duration + // attemptTimeout bounds each subscribe attempt and unsubscribe. + attemptTimeout time.Duration stripSubscribeFromBlockZero bool @@ -61,7 +63,11 @@ type Adapter struct { // resubCancel cancels the resubscribe loop of connection resubEpoch. resubCancel context.CancelFunc resubEpoch uint64 - heads upstreamSub + // resubDone is set once that loop has established everything; + // resubAgain asks a running loop for another pass before it finishes. + resubDone bool + resubAgain bool + heads upstreamSub // filters (by filterKey) survive disconnects to be resubscribed. filters map[string]*filterSub } @@ -110,6 +116,7 @@ func New(up *upstream.Upstream, networkID string, logger *zerolog.Logger, opts O }, retryMin: resubRetryMin, retryMax: resubRetryMax, + attemptTimeout: resubAttemptTimeout, stripSubscribeFromBlockZero: opts.StripSubscribeFromBlockZero, } } @@ -130,6 +137,8 @@ func (a *Adapter) Start(_ context.Context, nw indexer.NetworkHandle, sink indexe a.nw = nw a.sink = sink + // One adapter per client: each project has its own clients, an + // upstream belongs to one network, and a network is bootstrapped once. cbID := a.Name() a.wsClient.SetOnReconnect(cbID, func() { a.logger.Info().Msg("upstream websocket reconnected, resubscribing") @@ -143,6 +152,7 @@ func (a *Adapter) Start(_ context.Context, nw indexer.NetworkHandle, sink indexe if a.resubCancel != nil { a.resubCancel() } + a.resubDone = false a.heads.id = "" for _, sub := range a.filters { sub.id = "" @@ -160,7 +170,11 @@ func (a *Adapter) startResubscribe() { epoch := a.wsClient.Epoch() a.subsMu.Lock() defer a.subsMu.Unlock() - if epoch == 0 || epoch == a.resubEpoch { + if epoch == 0 { + return + } + if epoch == a.resubEpoch && !a.resubDone { + a.resubAgain = true return } if a.resubCancel != nil { @@ -168,12 +182,13 @@ func (a *Adapter) startResubscribe() { } ctx, cancel := context.WithCancel(context.Background()) a.resubCancel, a.resubEpoch = cancel, epoch + a.resubDone, a.resubAgain = false, false go a.resubscribeWithRetry(ctx) } -// EnsureFilter subscribes a filter; a no-op if it is already subscribed. A -// filter this call added is dropped again on failure, so it isn't -// resubscribed on every reconnect. +// EnsureFilter subscribes a filter; a no-op if it is already subscribed. On +// failure the filter is kept and retried in the background, like every +// other subscription of this upstream, until RemoveFilter. func (a *Adapter) EnsureFilter(ctx context.Context, subType, paramsHash string, params []interface{}) error { key := filterKey(subType, paramsHash) @@ -185,20 +200,30 @@ func (a *Adapter) EnsureFilter(ctx context.Context, subType, paramsHash string, } a.subsMu.Unlock() + // Bound the attempt: callers (the indexer's reconcile, client + // subscribes) hold locks across it, and a reply the upstream drops + // while its connection stays busy would otherwise never time out. A + // timed-out filter keeps retrying in the background. + ctx, cancel := context.WithTimeout(ctx, a.attemptTimeout) + defer cancel() sub.mu.Lock() - defer sub.mu.Unlock() err := a.subscribeFilterLocked(ctx, sub) - if err != nil && !exists { - a.subsMu.Lock() - if a.filters[key] == sub { - delete(a.filters, key) - } - a.subsMu.Unlock() - a.drop(ctx, &sub.upstreamSub) + sub.mu.Unlock() + if err != nil { + a.startResubscribe() } return err } +// FilterLive reports whether the filter's subscription is live on the +// current connection. +func (a *Adapter) FilterLive(subType, paramsHash string) bool { + a.subsMu.Lock() + defer a.subsMu.Unlock() + sub, ok := a.filters[filterKey(subType, paramsHash)] + return ok && sub.id != "" +} + // RemoveFilter unsubscribes a filter and stops resubscribing it. func (a *Adapter) RemoveFilter(ctx context.Context, subType, paramsHash string) error { key := filterKey(subType, paramsHash) @@ -228,7 +253,7 @@ func (a *Adapter) resubscribeWithRetry(ctx context.Context) { return } done := true - attemptCtx, cancel := context.WithTimeout(ctx, resubAttemptTimeout) + attemptCtx, cancel := context.WithTimeout(ctx, a.attemptTimeout) a.heads.mu.Lock() err := a.subscribeLocked(attemptCtx, &a.heads, []interface{}{indexer.SubTypeNewHeads}, a.handleNewHeads) a.heads.mu.Unlock() @@ -245,7 +270,7 @@ func (a *Adapter) resubscribeWithRetry(ctx context.Context) { } a.subsMu.Unlock() for _, sub := range filters { - attemptCtx, cancel := context.WithTimeout(ctx, resubAttemptTimeout) + attemptCtx, cancel := context.WithTimeout(ctx, a.attemptTimeout) sub.mu.Lock() err := a.subscribeFilterLocked(attemptCtx, sub) sub.mu.Unlock() @@ -257,6 +282,16 @@ func (a *Adapter) resubscribeWithRetry(ctx context.Context) { } } if done { + a.subsMu.Lock() + again := a.resubAgain && ctx.Err() == nil + a.resubAgain = false + if !again && ctx.Err() == nil { + a.resubDone = true + } + a.subsMu.Unlock() + if again { + continue + } a.logger.Info().Msg("all upstream subscriptions (re)established") return } @@ -323,7 +358,7 @@ func (a *Adapter) subscribeLocked(ctx context.Context, sub *upstreamSub, params return errNotConnected } if !commit { - a.release(context.WithoutCancel(ctx), ws.ID, ws.Epoch) + a.release(ctx, ws.ID, ws.Epoch) return nil } a.logger.Info().Str("upstreamSubId", ws.ID).Interface("subType", params[0]).Msg("subscribed upstream") @@ -344,9 +379,19 @@ func (a *Adapter) drop(ctx context.Context, sub *upstreamSub) { // release drops a subscription's handler and cancels it upstream. Both are // no-ops once its connection is gone, since the subscription died with it. +// The cancel outlives a caller context that is already done (a timed-out +// subscribe, a client that disconnected after unsubscribing), but a caller +// deadline still in the future and shorter than attemptTimeout bounds +// it, so teardown keeps its budget. func (a *Adapter) release(ctx context.Context, id string, epoch uint64) { a.wsClient.UnregisterSubscriptionHandler(id, epoch) - ctx, cancel := context.WithTimeout(ctx, resubAttemptTimeout) + timeout := a.attemptTimeout + if deadline, ok := ctx.Deadline(); ok { + if remaining := time.Until(deadline); remaining > 0 && remaining < timeout { + timeout = remaining + } + } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeout) defer cancel() _ = a.send(clients.WithWsSubscription(ctx, &clients.WsSubscription{Epoch: epoch}), methodEthUnsubscribe, []interface{}{id}) } diff --git a/indexer/adapters/wsupstream/adapter_reconnect_test.go b/indexer/adapters/wsupstream/adapter_reconnect_test.go index ec46a84c7..e133b3ac8 100644 --- a/indexer/adapters/wsupstream/adapter_reconnect_test.go +++ b/indexer/adapters/wsupstream/adapter_reconnect_test.go @@ -174,8 +174,9 @@ func newTestAdapter(t *testing.T, u *url.URL) (*Adapter, *fakeSink) { forward: func(ctx context.Context, nq *common.NormalizedRequest, _ bool) (*common.NormalizedResponse, error) { return wsc.SendRequest(ctx, nq) }, - retryMin: resubRetryMin, - retryMax: resubRetryMax, + retryMin: resubRetryMin, + retryMax: resubRetryMax, + attemptTimeout: resubAttemptTimeout, } return a, &fakeSink{events: make(chan indexer.StreamEvent, 64)} } @@ -353,7 +354,8 @@ func TestAdapterRemoveFilterDuringSubscribe(t *testing.T) { } // TestAdapterEnsureFilterWhileDisconnected: EnsureFilter must report the -// failure (so the indexer tries other ingresses) and not keep the filter. +// failure (so the indexer tries other ingresses) and keep the filter, to +// subscribe it once connected. func TestAdapterEnsureFilterWhileDisconnected(t *testing.T) { server := newNotifyServer(t) u := server.wsURL(t) @@ -361,9 +363,198 @@ func TestAdapterEnsureFilterWhileDisconnected(t *testing.T) { a, _ := newTestAdapter(t, u) params := logsParams("0xabc") - err := a.EnsureFilter(context.Background(), indexer.SubTypeLogs, indexer.BuildParamsKey(params), params) + hash := indexer.BuildParamsKey(params) + err := a.EnsureFilter(context.Background(), indexer.SubTypeLogs, hash, params) require.ErrorIs(t, err, errNotConnected) + assert.False(t, a.FilterLive(indexer.SubTypeLogs, hash)) a.subsMu.Lock() - assert.Empty(t, a.filters) + assert.Len(t, a.filters, 1) a.subsMu.Unlock() } + +// TestAdapterRetriesFailedEnsureFilter: a filter whose subscribe failed on +// a live connection is retried in the background until it is live, and +// RemoveFilter ends that. +func TestAdapterRetriesFailedEnsureFilter(t *testing.T) { + compressResubRetry(t) + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + <-server.newConn + require.NoError(t, a.Start(context.Background(), fakeNetworkHandle{}, sink)) + require.Eventually(t, subscribedHeads(a), 3*time.Second, 10*time.Millisecond) + + var failuresLeft atomic.Int64 + failuresLeft.Store(2) + send := a.forward + a.forward = func(ctx context.Context, nq *common.NormalizedRequest, bypass bool) (*common.NormalizedResponse, error) { + if m, _ := nq.Method(); m == methodEthSubscribe && failuresLeft.Add(-1) >= 0 { + return nil, errors.New("circuit breaker is open on upstream-level") + } + return send(ctx, nq, bypass) + } + + params := logsParams("0xabc") + hash := indexer.BuildParamsKey(params) + require.Error(t, a.EnsureFilter(context.Background(), indexer.SubTypeLogs, hash, params)) + require.Eventually(t, func() bool { return a.FilterLive(indexer.SubTypeLogs, hash) }, + 3*time.Second, 10*time.Millisecond, "the failed filter must be retried until live") + + require.NoError(t, a.RemoveFilter(context.Background(), indexer.SubTypeLogs, hash)) + assert.False(t, a.FilterLive(indexer.SubTypeLogs, hash)) +} + +// Cancelling a subscription upstream must not depend on the caller's +// context: a client that unsubscribes and disconnects at once still releases +// the upstream subscription. +func TestAdapterRemoveFilterWithCancelledContextUnsubscribes(t *testing.T) { + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + conn := <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + // Like the real path, refuse to send once the context is done. + send := a.forward + a.forward = func(ctx context.Context, nq *common.NormalizedRequest, bypass bool) (*common.NormalizedResponse, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return send(ctx, nq, bypass) + } + + params := logsParams("0xabc") + hash := indexer.BuildParamsKey(params) + require.NoError(t, a.EnsureFilter(context.Background(), indexer.SubTypeLogs, hash, params)) + subID := conn.nextRequest(t, methodEthSubscribe).SubID + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, a.RemoveFilter(ctx, indexer.SubTypeLogs, hash)) + unsub := conn.nextRequest(t, methodEthUnsubscribe) + assert.Equal(t, []interface{}{subID}, unsub.Params) +} + +// A caller deadline still in the future bounds the upstream cancel, so +// connection teardown keeps its budget when the upstream stops answering. +func TestAdapterReleaseHonoursLiveCallerDeadline(t *testing.T) { + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + conn := <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + + params := logsParams("0xabc") + hash := indexer.BuildParamsKey(params) + require.NoError(t, a.EnsureFilter(context.Background(), indexer.SubTypeLogs, hash, params)) + _ = conn.nextRequest(t, methodEthSubscribe) + + // The upstream never answers the unsubscribe. + a.forward = func(ctx context.Context, _ *common.NormalizedRequest, _ bool) (*common.NormalizedResponse, error) { + <-ctx.Done() + return nil, ctx.Err() + } + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + start := time.Now() + require.NoError(t, a.RemoveFilter(ctx, indexer.SubTypeLogs, hash)) + assert.Less(t, time.Since(start), 2*time.Second, "release must stop at the caller's deadline, not the attempt timeout") +} + +func failFirstUnsubscribe(a *Adapter) *atomic.Bool { + var failed atomic.Bool + send := a.forward + a.forward = func(ctx context.Context, nq *common.NormalizedRequest, bypass bool) (*common.NormalizedResponse, error) { + if m, _ := nq.Method(); m == methodEthUnsubscribe && failed.CompareAndSwap(false, true) { + return nil, errors.New("upstream-level rate limit exceeded") // never reaches the wire + } + return send(ctx, nq, bypass) + } + return &failed +} + +// RemoveFilter's unsubscribe fails while the connection stays up; the +// still-live upstream subscription must be cancelled when it next notifies. +func TestAdapterFailedUnsubscribeIsRetried(t *testing.T) { + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + conn := <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + failed := failFirstUnsubscribe(a) + + params := logsParams("0xabc") + hash := indexer.BuildParamsKey(params) + require.NoError(t, a.EnsureFilter(context.Background(), indexer.SubTypeLogs, hash, params)) + subID := conn.nextRequest(t, methodEthSubscribe).SubID + nextEvent(t, sink, indexer.KindLog) + + require.NoError(t, a.RemoveFilter(context.Background(), indexer.SubTypeLogs, hash)) + require.True(t, failed.Load()) + + conn.sendLog(subID, 2) // the upstream subscription is still live + unsub := conn.nextRequest(t, methodEthUnsubscribe) + assert.Equal(t, []interface{}{subID}, unsub.Params) + select { + case ev := <-sink.events: + t.Fatalf("removed filter delivered %+v", ev) + default: + } +} + +// The same, for an in-flight subscribe whose result is released because its +// filter was removed meanwhile. +func TestAdapterFailedReleaseOfUncommittedSubscribeIsRetried(t *testing.T) { + server := newNotifyServer(t) + gate := make(chan struct{}) + server.gate = gate + a, sink := newTestAdapter(t, server.wsURL(t)) + conn := <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + failed := failFirstUnsubscribe(a) + + params := logsParams("0xabc") + hash := indexer.BuildParamsKey(params) + done := make(chan error, 1) + go func() { done <- a.EnsureFilter(context.Background(), indexer.SubTypeLogs, hash, params) }() + subID := conn.nextRequest(t, methodEthSubscribe).SubID + require.NoError(t, a.RemoveFilter(context.Background(), indexer.SubTypeLogs, hash)) + close(gate) + require.NoError(t, <-done) + require.True(t, failed.Load()) + + conn.sendLog(subID, 2) + unsub := conn.nextRequest(t, methodEthUnsubscribe) + assert.Equal(t, []interface{}{subID}, unsub.Params) +} + +// A filter subscribe whose reply never arrives must still return: callers +// hold locks across it (the indexer's reconcile, client subscribes). +func TestAdapterEnsureFilterIsBounded(t *testing.T) { + server := newNotifyServer(t) + a, sink := newTestAdapter(t, server.wsURL(t)) + a.attemptTimeout = 300 * time.Millisecond + // The failed subscribe starts a background retry that outlives the test. + nop := zerolog.Nop() + a.logger = &nop + t.Cleanup(func() { + a.subsMu.Lock() + if a.resubCancel != nil { + a.resubCancel() + } + a.subsMu.Unlock() + }) + <-server.newConn + require.Eventually(t, a.wsClient.IsConnected, 3*time.Second, 10*time.Millisecond) + a.sink = sink + // The upstream never answers the subscribe. + a.forward = func(ctx context.Context, _ *common.NormalizedRequest, _ bool) (*common.NormalizedResponse, error) { + <-ctx.Done() + return nil, ctx.Err() + } + + params := logsParams("0xabc") + start := time.Now() + err := a.EnsureFilter(context.Background(), indexer.SubTypeLogs, indexer.BuildParamsKey(params), params) + require.Error(t, err) + assert.Less(t, time.Since(start), 3*time.Second, "EnsureFilter must stop at the attempt timeout") +} diff --git a/indexer/indexer.go b/indexer/indexer.go index 11a592f3e..2b408c79b 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "github.com/rs/zerolog" ) @@ -50,6 +51,9 @@ type networkState struct { filterMu sync.RWMutex filters map[string]*filterState // paramsHash -> state + + // reconciling is set while a reconcile of the filters runs. + reconciling atomic.Bool } // filterState is one filter subscription shared by every client with the @@ -61,6 +65,12 @@ type filterState struct { subscribed bool // guarded by mu refs int // guarded by networkState.filterMu dedup *DedupWindow + + // subType and params are the filter's; on names the ingresses it was + // put on and not removed from since. Guarded by mu. + subType string + params []interface{} + on map[string]struct{} } // New returns an empty Indexer. Events for networks not registered with @@ -147,7 +157,8 @@ func (i *Indexer) EnsureFilter(ctx context.Context, networkId, subType string, p if f.subscribed { return paramsHash, nil } - if err := i.subscribe(ctx, ns, subType, paramsHash, params); err != nil { + f.subType, f.params, f.on = subType, params, make(map[string]struct{}) + if err := i.subscribe(ctx, ns, f, paramsHash); err != nil { // Ingresses may keep a filter whose subscribe failed (to retry on // reconnect), so remove it everywhere. i.removeFromIngresses(ctx, ns, subType, paramsHash) @@ -163,30 +174,24 @@ func (i *Indexer) EnsureFilter(ctx context.Context, networkId, subType string, p return paramsHash, nil } -// subscribe tries the selector's defaults, then its fallbacks only if every -// default failed. It fails when no ingress subscribed, including when none -// was selected. -func (i *Indexer) subscribe(ctx context.Context, ns *networkState, subType, paramsHash string, params []interface{}) error { +// subscribe puts a new filter on the selector's defaults, and on its +// fallbacks if no default took it. It fails when no ingress subscribed, +// including when none was selected. An ingress that failed keeps retrying +// the filter until it is removed. The caller holds f.mu. +func (i *Indexer) subscribe(ctx context.Context, ns *networkState, f *filterState, paramsHash string) error { networkId := ns.handle.Id() - - ns.ingressMu.RLock() - ings := make(map[string]EventIngress, len(ns.ingresses)) - for name, ing := range ns.ingresses { - ings[name] = ing - } - sel := ns.selector - ns.ingressMu.RUnlock() - - defaults, fallbacks := partitionIngresses(sel, ings, networkId, subType, params) + ings, sel := ns.snapshotIngresses() + defaults, fallbacks := partitionIngresses(sel, ings, networkId, f.subType, f.params) var errs []error for _, tier := range [][]EventIngress{defaults, fallbacks} { subscribed := false for _, ing := range tier { - if err := ing.EnsureFilter(ctx, subType, paramsHash, params); err != nil { + f.on[ing.Name()] = struct{}{} + if err := ing.EnsureFilter(ctx, f.subType, paramsHash, f.params); err != nil { errs = append(errs, err) i.logger.Warn().Err(err).Str("ingress", ing.Name()).Str("networkId", networkId). - Str("subType", subType).Str("paramsHash", paramsHash). + Str("subType", f.subType).Str("paramsHash", paramsHash). Msg("ingress EnsureFilter failed") continue } @@ -197,11 +202,99 @@ func (i *Indexer) subscribe(ctx context.Context, ns *networkState, subType, para } } if len(errs) == 0 { - return fmt.Errorf("indexer: no ingress selected for %s filter on network %q", subType, networkId) + return fmt.Errorf("indexer: no ingress selected for %s filter on network %q", f.subType, networkId) } return errors.Join(errs...) } +// snapshotIngresses copies the network's ingresses and selector. +func (ns *networkState) snapshotIngresses() (map[string]EventIngress, IngressSelector) { + ns.ingressMu.RLock() + defer ns.ingressMu.RUnlock() + ings := make(map[string]EventIngress, len(ns.ingresses)) + for name, ing := range ns.ingresses { + ings[name] = ing + } + return ings, ns.selector +} + +// triggerReconcile starts a reconcile of ns's filters unless one is running +// or there are none. Triggers it skips are covered by the next one. +func (i *Indexer) triggerReconcile(ns *networkState) { + ns.filterMu.RLock() + none := len(ns.filters) == 0 + ns.filterMu.RUnlock() + if none || !ns.reconciling.CompareAndSwap(false, true) { + return + } + go func() { + defer ns.reconciling.Store(false) + i.reconcile(context.Background(), ns) + }() +} + +// reconcile moves every subscribed filter of ns onto the ingresses the +// selector picks now: all its defaults, plus its fallbacks while no default +// has the filter live; it is removed from every other ingress it is on. +func (i *Indexer) reconcile(ctx context.Context, ns *networkState) { + ns.filterMu.RLock() + filters := make(map[string]*filterState, len(ns.filters)) + for hash, f := range ns.filters { + filters[hash] = f + } + ns.filterMu.RUnlock() + ings, sel := ns.snapshotIngresses() + + for hash, f := range filters { + f.mu.Lock() + if f.subscribed { + i.reconcileFilter(ctx, ns, ings, sel, hash, f) + } + f.mu.Unlock() + } +} + +// reconcileFilter is reconcile for one filter; the caller holds f.mu. +func (i *Indexer) reconcileFilter(ctx context.Context, ns *networkState, ings map[string]EventIngress, sel IngressSelector, hash string, f *filterState) { + defaults, fallbacks := partitionIngresses(sel, ings, ns.handle.Id(), f.subType, f.params) + want := make(map[string]EventIngress, len(defaults)+len(fallbacks)) + covered := false + for _, ing := range defaults { + want[ing.Name()] = ing + covered = covered || ing.FilterLive(f.subType, hash) + } + if !covered { + for _, ing := range fallbacks { + want[ing.Name()] = ing + } + } + + for name, ing := range want { + if _, ok := f.on[name]; ok { + continue + } + f.on[name] = struct{}{} + if err := ing.EnsureFilter(ctx, f.subType, hash, f.params); err != nil { + i.logger.Warn().Err(err).Str("ingress", name).Str("networkId", ns.handle.Id()). + Str("subType", f.subType).Str("paramsHash", hash). + Msg("ingress EnsureFilter failed, it keeps retrying") + } + } + for name := range f.on { + if _, ok := want[name]; ok { + continue + } + delete(f.on, name) + if ing := ings[name]; ing != nil { + if err := ing.RemoveFilter(ctx, f.subType, hash); err != nil { + i.logger.Warn().Err(err).Str("ingress", name).Str("networkId", ns.handle.Id()). + Str("subType", f.subType).Str("paramsHash", hash). + Msg("ingress RemoveFilter failed") + } + } + } +} + // partitionIngresses resolves the selector's tiers to registered ingresses, // dropping unknown and repeated names. Without a selector every ingress is // a default. @@ -264,7 +357,7 @@ func (i *Indexer) ReleaseFilter(ctx context.Context, networkId, subType, paramsH return } i.removeFromIngresses(ctx, ns, subType, paramsHash) - f.subscribed = false + f.subscribed, f.on = false, nil ns.filterMu.Lock() if f.refs == 0 { delete(ns.filters, paramsHash) @@ -300,6 +393,11 @@ func (i *Indexer) Ingest(ev StreamEvent) { } ns := nsRaw.(*networkState) + // Each head, delivered or not, rechecks where the filters belong. + if ev.Kind == KindNewHead { + i.triggerReconcile(ns) + } + // Before dedup, so every source's head counts even if another source // already delivered it. if ev.Kind == KindNewHead && !ev.Block.Zero() && ev.SourceId != "" { diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go index 3d0866c88..08c98851c 100644 --- a/indexer/indexer_test.go +++ b/indexer/indexer_test.go @@ -114,6 +114,9 @@ func (i *fakeIngress) EnsureFilter(_ context.Context, _ string, paramsHash strin i.active.Store(true) return i.getErr() } +func (i *fakeIngress) FilterLive(_, _ string) bool { + return i.active.Load() && i.getErr() == nil +} func (i *fakeIngress) RemoveFilter(_ context.Context, _, _ string) error { if i.hook != nil { i.hook("remove") @@ -682,6 +685,124 @@ func TestIndexer_Selector_UnnamedIngressIsExcluded(t *testing.T) { } } +// reconcileNow runs a reconcile of evm:1 synchronously. +func reconcileNow(t *testing.T, idx *Indexer) { + t.Helper() + nsRaw, ok := idx.networks.Load("evm:1") + if !ok { + t.Fatal("evm:1 not registered") + } + idx.reconcile(context.Background(), nsRaw.(*networkState)) +} + +// The fallbacks carry a filter only while no default has it live: they take +// over when the defaults lose it, and hand it back once one has it again. +func TestIndexer_Reconcile_FallbackTakesOverAndHandsBack(t *testing.T) { + idx := newIndexer(t) + a, b, c := registerThreeIngresses(t, idx) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{defaults: []string{"a", "b"}, fallbacks: []string{"c"}}) + if _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}); err != nil { + t.Fatal(err) + } + + reconcileNow(t, idx) + if c.ensureCalls.Load() != 0 { + t.Fatalf("fallback must stay idle while a default is live, got %d calls", c.ensureCalls.Load()) + } + + a.setErr(errTest("a lost it")) + b.setErr(errTest("b lost it")) + reconcileNow(t, idx) + if c.ensureCalls.Load() != 1 || !c.FilterLive("logs", "") { + t.Fatalf("fallback must take over once no default is live, got %d calls", c.ensureCalls.Load()) + } + if a.removeCalls.Load() != 0 || b.removeCalls.Load() != 0 { + t.Fatal("defaults keep the filter, to retry it") + } + + reconcileNow(t, idx) + if c.ensureCalls.Load() != 1 { + t.Fatalf("an ingress already carrying the filter must not be asked again, got %d", c.ensureCalls.Load()) + } + + b.setErr(nil) + reconcileNow(t, idx) + if c.removeCalls.Load() != 1 || c.active.Load() { + t.Fatalf("fallback must hand the filter back once a default is live, removes=%d", c.removeCalls.Load()) + } +} + +// A default the selector no longer names (e.g. the policy excluded it) gives +// the filter up; if that leaves no default, the fallbacks take it. +func TestIndexer_Reconcile_FollowsSelector(t *testing.T) { + idx := newIndexer(t) + a, b, c := registerThreeIngresses(t, idx) + sel := &fakeSelector{defaults: []string{"a", "b"}, fallbacks: []string{"c"}} + idx.RegisterNetworkSelector("evm:1", sel) + if _, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}); err != nil { + t.Fatal(err) + } + + sel.defaults = []string{"b"} + reconcileNow(t, idx) + if a.removeCalls.Load() != 1 || c.ensureCalls.Load() != 0 { + t.Fatalf("a must give the filter up while b still carries it: a.removes=%d c.ensures=%d", + a.removeCalls.Load(), c.ensureCalls.Load()) + } + + sel.defaults = nil + reconcileNow(t, idx) + if b.removeCalls.Load() != 1 || c.ensureCalls.Load() != 1 { + t.Fatalf("with no default left the fallback must carry it: b.removes=%d c.ensures=%d", + b.removeCalls.Load(), c.ensureCalls.Load()) + } + + // Returning defaults take it back; the fallback keeps it until one of + // them has it live, so there is no gap. + sel.defaults = []string{"a", "b"} + reconcileNow(t, idx) + if a.ensureCalls.Load() != 2 || b.ensureCalls.Load() != 2 || c.removeCalls.Load() != 0 { + t.Fatalf("returning defaults must take it back alongside the fallback: a=%d b=%d c.removes=%d", + a.ensureCalls.Load(), b.ensureCalls.Load(), c.removeCalls.Load()) + } + reconcileNow(t, idx) + if c.removeCalls.Load() != 1 { + t.Fatalf("the fallback must hand back once a default is live, removes=%d", c.removeCalls.Load()) + } +} + +// Every head triggers the recheck, and a released filter is left alone. +func TestIndexer_Reconcile_RunsOnHeadsAndSkipsReleased(t *testing.T) { + idx := newIndexer(t) + a, _, c := registerThreeIngresses(t, idx) + idx.RegisterNetworkSelector("evm:1", &fakeSelector{defaults: []string{"a"}, fallbacks: []string{"c"}}) + h, err := idx.EnsureFilter(context.Background(), "evm:1", "logs", []interface{}{"logs"}) + if err != nil { + t.Fatal(err) + } + a.setErr(errTest("a lost it")) + + idx.Ingest(StreamEvent{Kind: KindNewHead, NetworkId: "evm:1", SourceId: "ws:c", Block: BlockRef{Number: 1, Hash: "0x1"}}) + deadline := time.Now().Add(2 * time.Second) + for c.ensureCalls.Load() != 1 { + if time.Now().After(deadline) { + t.Fatal("a head must trigger the recheck") + } + time.Sleep(5 * time.Millisecond) + } + + nsRaw, _ := idx.networks.Load("evm:1") + ns := nsRaw.(*networkState) + for ns.reconciling.Load() { + time.Sleep(time.Millisecond) + } + idx.ReleaseFilter(context.Background(), "evm:1", "logs", h) + reconcileNow(t, idx) + if a.ensureCalls.Load() != 1 || c.ensureCalls.Load() != 1 { + t.Fatalf("a released filter must not be resubscribed: a=%d c=%d", a.ensureCalls.Load(), c.ensureCalls.Load()) + } +} + func TestIndexer_Selector_NothingSelectedFails(t *testing.T) { idx := newIndexer(t) registerThreeIngresses(t, idx) diff --git a/indexer/ingress.go b/indexer/ingress.go index 26338e672..ec72629ff 100644 --- a/indexer/ingress.go +++ b/indexer/ingress.go @@ -32,12 +32,15 @@ type EventIngress interface { // RemoveFilter unsubscribes the filter; a no-op if it was never // subscribed. Called when its last client leaves or its subscribe failed. RemoveFilter(ctx context.Context, subType string, paramsHash string) error + // FilterLive reports whether the filter is subscribed and live now. + FilterLive(subType string, paramsHash string) bool } // IngressSelector chooses which ingresses carry a filter subscription, by -// Name(). Defaults are tried first and at least one must succeed; fallbacks -// are tried only if every default failed. Ingresses named in neither are -// excluded. Without a selector every ingress is a default. +// Name(). A filter is kept on every default, and on the fallbacks while no +// default has it live (or, when it is first subscribed, if every default +// failed). Ingresses named in neither do not carry it. The choice is +// rechecked on every head. Without a selector every ingress is a default. type IngressSelector interface { Select(networkId, subType string, params []interface{}) (defaults, fallbacks []string) } From b5a30a3a0aa904ed52ab5c5a050dc2249d6dc885 Mon Sep 17 00:00:00 2001 From: shpookas Date: Fri, 9 Oct 2026 11:02:19 +0200 Subject: [PATCH 15/21] feat(evm): opt-in short-circuit for future blocks without served-tip tryShortCircuitFutureBlock returns null for a numbered eth_getBlockByNumber above the highest head any eligible upstream has (static caps included), so it never nulls a block an upstream has. It only runs when served-tip is enabled for latest. Clients that poll the next block by number get a null from each upstream in turn on networks without served-tip, so every poll walks every routed upstream, fallbacks included, and the retry layer repeats the walk. Add evm.shortCircuitFutureBlocks to enable the short-circuit on its own, without changing what latest resolves to. It is off by default: the head is what eRPC last saw, so a block produced since then can briefly get null (the issue 934 case), which only suits networks whose heads arrive promptly. --- common/config.go | 23 ++++++++ common/defaults.go | 3 + docs/pages/config/projects/networks.mdx | 46 +++++++++++++++ erpc/networks.go | 7 ++- erpc/networks_future_block_test.go | 75 +++++++++++++++++++++++++ typescript/config/src/generated.ts | 10 ++++ 6 files changed, 161 insertions(+), 3 deletions(-) diff --git a/common/config.go b/common/config.go index 58b5e238f..4e15f8e16 100644 --- a/common/config.go +++ b/common/config.go @@ -2681,6 +2681,15 @@ type EvmNetworkConfig struct { // finalized head; an unfinalized block's empty is treated as not-yet-confirmed. EmptyResultConfidence AvailbilityConfidence `yaml:"emptyResultConfidence,omitempty" json:"emptyResultConfidence,omitempty"` + // ShortCircuitFutureBlocks returns null for a numbered eth_getBlockByNumber + // above the highest head any eligible upstream has, without dispatching it. + // Clients that poll the next block by number otherwise walk every routed + // upstream on each poll. Always on when servedTip is enabled for "latest". + // The head is what the state poller and head subscriptions last reported, so + // a block an upstream produced since then gets null until eRPC sees it: only + // enable it where heads arrive promptly (e.g. over WebSocket). + ShortCircuitFutureBlocks *bool `yaml:"shortCircuitFutureBlocks,omitempty" json:"shortCircuitFutureBlocks,omitempty"` + // SafeBlockSource is an upstream id/tag selector for standard JSON-RPC // requests carrying the `safe` block tag. Matching upstreams define and // serve `safe`; empty (without an inherited network default) keeps existing @@ -2761,6 +2770,20 @@ type EvmServedTipConfig struct { TrajectoryWindow *Duration `yaml:"trajectoryWindow,omitempty" json:"trajectoryWindow,omitempty"` } +// ShortCircuitFutureBlocksEnabled reports whether a numbered eth_getBlockByNumber +// above every eligible upstream's head returns null without dispatching: when +// ShortCircuitFutureBlocks is set, or servedTip is enabled for "latest". +// Nil-receiver safe. +func (c *EvmNetworkConfig) ShortCircuitFutureBlocksEnabled() bool { + if c == nil { + return false + } + if c.ShortCircuitFutureBlocks != nil && *c.ShortCircuitFutureBlocks { + return true + } + return c.ServedTipEnabledFor("latest") +} + // ServedTipEnabledFor reports whether the majority served tip is enabled for // the given block axis ("latest" or "finalized"). The "safe" tag resolves to the // finalized axis, so listing "safe" in EnabledFor enables it for "finalized". diff --git a/common/defaults.go b/common/defaults.go index ba247afac..9d08d3bec 100644 --- a/common/defaults.go +++ b/common/defaults.go @@ -2280,6 +2280,9 @@ func (n *NetworkConfig) SetDefaults(upstreams []*UpstreamConfig, defaults *Netwo if n.Evm.EmptyResultConfidence == 0 && defaults.Evm.EmptyResultConfidence != 0 { n.Evm.EmptyResultConfidence = defaults.Evm.EmptyResultConfidence } + if n.Evm.ShortCircuitFutureBlocks == nil && defaults.Evm.ShortCircuitFutureBlocks != nil { + n.Evm.ShortCircuitFutureBlocks = defaults.Evm.ShortCircuitFutureBlocks + } if n.Evm.StripSubscribeFromBlockZero == nil && defaults.Evm.StripSubscribeFromBlockZero != nil { n.Evm.StripSubscribeFromBlockZero = defaults.Evm.StripSubscribeFromBlockZero } diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index aa3089462..9dbfe1165 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -99,6 +99,52 @@ export default createConfig({ Leave unset for chains where upstreams handle `fromBlock: "0x0"` in an `eth_subscribe` filter correctly (most do — the flag is a targeted workaround, not a universal default). It can also be set once for every network under `networkDefaults.evm`. +## Future-block polling + +Clients that follow a chain by number (for example rollup nodes reading L1) poll `eth_getBlockByNumber` for the block after the head and treat `null` as "not produced yet". No upstream has that block, so each one returns `null` in turn and every poll walks every routed upstream, fallbacks included. Enable `shortCircuitFutureBlocks` to return `null` for a numbered `eth_getBlockByNumber` above the highest head any eligible upstream has (static availability caps included), without dispatching it. A block any eligible upstream has is still dispatched. It is always on when `servedTip.enabledFor` includes `"latest"`. + + + +```yaml filename="erpc.yaml" +projects: + - id: main + networks: + - architecture: evm + evm: + chainId: 1 + # Returns null for eth_getBlockByNumber(N) when N is above every + # eligible upstream's head, instead of asking each upstream. + shortCircuitFutureBlocks: true # default: false +``` + + +```ts filename="erpc.ts" +import { createConfig } from "@erpc-cloud/config"; + +export default createConfig({ + projects: [ + { + id: "main", + networks: [ + { + architecture: "evm", + evm: { + chainId: 1, + // Returns null for eth_getBlockByNumber(N) when N is above every + // eligible upstream's head, instead of asking each upstream. + shortCircuitFutureBlocks: true, // default: false + }, + }, + ], + }, + ], +}); +``` + + + +The head is what eRPC last saw from the state poller and head subscriptions. A block an upstream produced after that gets `null` until eRPC sees it, so a client that learned of the block elsewhere (for example from the node's own `newHeads`) can briefly get `null` for a block that exists. Only enable it where heads arrive promptly, such as networks with WebSocket head subscriptions, and where clients retry a `null`. + ## Failover By default a request only reaches the upstreams the [selection policy](/config/projects/selection-policies) currently routes to; upstreams tagged `tier:fallback` join once the policy promotes them, which happens on its evaluation interval. `failover.onDefaultsExhausted` adds a per-request escape for the moments before that: when every routed upstream failed with an error that another upstream might not repeat, the same request is retried once on the `tier:fallback` upstreams it has not tried yet — even while the policy still keeps them cordoned. diff --git a/erpc/networks.go b/erpc/networks.go index 9934a9300..678b761b0 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -891,13 +891,14 @@ func (n *Network) evmHeadReference(ctx context.Context, useFinalized bool) serve // // Safety: it compares against the highest effective head across eligible // upstreams, static caps included (servedTipReference.Available), so it never -// nulls out a block any upstream actually serves. It is gated on served-tip being enabled for the latest -// axis — the same opt-in that makes the head trustworthy — and the synthesized +// nulls out a block any upstream actually serves. It runs when the network sets +// evm.shortCircuitFutureBlocks or enables served-tip for the latest axis (see +// EvmNetworkConfig.ShortCircuitFutureBlocksEnabled), and the synthesized // response is returned directly from Forward, so it is never written to cache // (the block will exist later). The post-forward empty guard remains as // defense-in-depth for the in-flight case where an upstream advances mid-request. func (n *Network) tryShortCircuitFutureBlock(ctx context.Context, req *common.NormalizedRequest, method string) (*common.NormalizedResponse, bool) { - if n.cfg == nil || n.cfg.Evm == nil || !n.servedTipEnabledFor("latest") { + if n.cfg == nil || !n.cfg.Evm.ShortCircuitFutureBlocksEnabled() { return nil, false } if !strings.EqualFold(method, "eth_getBlockByNumber") { diff --git a/erpc/networks_future_block_test.go b/erpc/networks_future_block_test.go index 007a1659a..1a4cac4be 100644 --- a/erpc/networks_future_block_test.go +++ b/erpc/networks_future_block_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "strings" + "sync/atomic" "testing" "github.com/erpc/erpc/common" @@ -168,3 +169,77 @@ func TestForward_FutureBlock_LatestTag_Dispatches(t *testing.T) { assert.Contains(t, jrr.GetResultString(), "0x270f", "latest tag has no concrete future number; must dispatch normally") } + +// evm.shortCircuitFutureBlocks enables the short-circuit without served-tip: a +// block above every head returns null and reaches no upstream. +func TestForward_FutureBlock_OptInWithoutServedTip_ShortCircuitsToNull(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _ := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "fb1", chainID: 123, latestBlock: 100}, + {id: "fb2", chainID: 123, latestBlock: 99}, + {id: "fb3", chainID: 123, latestBlock: 98}, + }, nil) + network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) + var dispatched atomic.Int64 + for _, id := range []string{"fb1", "fb2", "fb3"} { + gock.New("http://" + id + ".localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + if !strings.Contains(util.SafeReadBody(r), "eth_getBlockByNumber") { + return false + } + dispatched.Add(1) + return true + }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":null}`)) + } + + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x69",false]}`)) + resp, err := network.Forward(ctx, req) + require.NoError(t, err) + require.NotNil(t, resp) + assert.True(t, resp.IsResultEmptyish(ctx), + "block 105 > max head 100 must short-circuit to null") + assert.Zero(t, dispatched.Load(), + "a block beyond every upstream's head must not be dispatched to any upstream") +} + +// Without served-tip "latest" is the corroborated (second-highest) head, but a +// block only the most-ahead upstream has is still servable: with the opt-in it +// must be dispatched, not nulled. +func TestForward_FutureBlock_OptInWithoutServedTip_OnlyMostAheadHasBlock_Dispatches(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _ := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "fb1", chainID: 123, latestBlock: 100}, + {id: "fb2", chainID: 123, latestBlock: 99}, + {id: "fb3", chainID: 123, latestBlock: 98}, + }, nil) + network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) + mockGetBlockByNumberNonNull("fb1", "fb2", "fb3") + + // block 100 (0x64) is above the corroborated head (99) but fb1 has it. + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x64",false]}`)) + resp, err := network.Forward(ctx, req) + require.NoError(t, err) + require.NotNil(t, resp) + jrr, err := resp.JsonRpcResponse(ctx) + require.NoError(t, err) + assert.Contains(t, jrr.GetResultString(), "0x270f", + "a block the most-ahead upstream has must be dispatched, not short-circuited") +} diff --git a/typescript/config/src/generated.ts b/typescript/config/src/generated.ts index 862d69821..f0d773d04 100644 --- a/typescript/config/src/generated.ts +++ b/typescript/config/src/generated.ts @@ -1702,6 +1702,16 @@ export interface EvmNetworkConfig { * finalized head; an unfinalized block's empty is treated as not-yet-confirmed. */ emptyResultConfidence?: AvailbilityConfidence; + /** + * ShortCircuitFutureBlocks returns null for a numbered eth_getBlockByNumber + * above the highest head any eligible upstream has, without dispatching it. + * Clients that poll the next block by number otherwise walk every routed + * upstream on each poll. Always on when servedTip is enabled for "latest". + * The head is what the state poller and head subscriptions last reported, so + * a block an upstream produced since then gets null until eRPC sees it: only + * enable it where heads arrive promptly (e.g. over WebSocket). + */ + shortCircuitFutureBlocks?: boolean; /** * SafeBlockSource is an upstream id/tag selector for standard JSON-RPC * requests carrying the `safe` block tag. Matching upstreams define and From 1dd15134e503dbaab6bb107875c4f8d7ed17671c Mon Sep 17 00:00:00 2001 From: shpookas Date: Fri, 9 Oct 2026 12:00:27 +0200 Subject: [PATCH 16/21] fix(evm): keep dispatchable syncing upstreams in the future-block check The future-block short-circuit compares against the head reference, which leaves syncing upstreams out. An upstream that reports syncing still receives requests unless evm.skipWhenSyncing is set, so it may already have a block above that reference. Dispatch when one of them is at or above the block. Also: - test networkDefaults inheritance and an explicit false override - register gock mocks before network setup in the new forwarding tests - docs: config schema row, edge cases, span attribute, comment style --- common/config.go | 2 +- common/config_test.go | 26 +++++ docs/pages/config/projects/networks.mdx | 17 +++- erpc/networks.go | 26 +++++ erpc/networks_future_block_test.go | 126 +++++++++++++++++++----- 5 files changed, 165 insertions(+), 32 deletions(-) diff --git a/common/config.go b/common/config.go index 4e15f8e16..226b4f46f 100644 --- a/common/config.go +++ b/common/config.go @@ -2772,7 +2772,7 @@ type EvmServedTipConfig struct { // ShortCircuitFutureBlocksEnabled reports whether a numbered eth_getBlockByNumber // above every eligible upstream's head returns null without dispatching: when -// ShortCircuitFutureBlocks is set, or servedTip is enabled for "latest". +// ShortCircuitFutureBlocks is true, or servedTip is enabled for "latest". // Nil-receiver safe. func (c *EvmNetworkConfig) ShortCircuitFutureBlocksEnabled() bool { if c == nil { diff --git a/common/config_test.go b/common/config_test.go index 9963e5f5f..89873afc6 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -1297,6 +1297,32 @@ func TestNetworkConfig_SetDefaults_StripSubscribeFromBlockZeroInheritsFromDefaul assert.Equal(t, &strip, n.Evm.StripSubscribeFromBlockZero) } +func TestNetworkConfig_SetDefaults_ShortCircuitFutureBlocksInheritsFromDefaults(t *testing.T) { + enabled, disabled := true, false + defaults := &NetworkDefaults{Evm: &EvmNetworkConfig{ShortCircuitFutureBlocks: &enabled}} + + inherited := &NetworkConfig{Architecture: ArchitectureEvm, Evm: &EvmNetworkConfig{ChainId: 1}} + assert.NoError(t, inherited.SetDefaults(nil, defaults)) + assert.True(t, inherited.Evm.ShortCircuitFutureBlocksEnabled()) + + // An explicit network-level false wins over a true default. + overridden := &NetworkConfig{Architecture: ArchitectureEvm, Evm: &EvmNetworkConfig{ChainId: 1, ShortCircuitFutureBlocks: &disabled}} + assert.NoError(t, overridden.SetDefaults(nil, defaults)) + assert.False(t, overridden.Evm.ShortCircuitFutureBlocksEnabled()) +} + +func TestEvmNetworkConfig_ShortCircuitFutureBlocksEnabled(t *testing.T) { + var nilCfg *EvmNetworkConfig + assert.False(t, nilCfg.ShortCircuitFutureBlocksEnabled()) + assert.False(t, (&EvmNetworkConfig{}).ShortCircuitFutureBlocksEnabled()) + enabled, disabled := true, false + assert.False(t, (&EvmNetworkConfig{ShortCircuitFutureBlocks: &disabled}).ShortCircuitFutureBlocksEnabled()) + assert.True(t, (&EvmNetworkConfig{ShortCircuitFutureBlocks: &enabled}).ShortCircuitFutureBlocksEnabled()) + // servedTip for "latest" keeps the short-circuit on, as before. + assert.True(t, (&EvmNetworkConfig{ServedTip: &EvmServedTipConfig{EnabledFor: []string{"latest"}}}).ShortCircuitFutureBlocksEnabled()) + assert.False(t, (&EvmNetworkConfig{ServedTip: &EvmServedTipConfig{EnabledFor: []string{"finalized"}}}).ShortCircuitFutureBlocksEnabled()) +} + func TestUpstreamConfig_ValidateRateLimitCountMode(t *testing.T) { cfg := &Config{} base := func(mode RateLimitCountMode) *UpstreamConfig { diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 9dbfe1165..29ee4b875 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -114,7 +114,8 @@ projects: chainId: 1 # Returns null for eth_getBlockByNumber(N) when N is above every # eligible upstream's head, instead of asking each upstream. - shortCircuitFutureBlocks: true # default: false + # default: false + shortCircuitFutureBlocks: true ``` @@ -132,7 +133,8 @@ export default createConfig({ chainId: 1, // Returns null for eth_getBlockByNumber(N) when N is above every // eligible upstream's head, instead of asking each upstream. - shortCircuitFutureBlocks: true, // default: false + // default: false + shortCircuitFutureBlocks: true, }, }, ], @@ -332,6 +334,7 @@ https://docs.erpc.cloud/config/projects/networks.llms.txt`} | `idempotentTransactionBroadcast` | `*bool` | `nil` = **enabled** () | "Already known"/"nonce too low"-verified errors become success-with-tx-hash, making retry/hedge safe for `eth_sendRawTransaction`. | | `markEmptyAsErrorMethods` | `[]string` | `eth_blockNumber`, `eth_getBlockByNumber`, `eth_getTransactionByHash`, and 8 others () | Methods where empty response = error (retried, upstream scored down). `eth_getTransactionReceipt` deliberately excluded. | | `emptyResultConfidence` | `blockHead` \| `finalizedBlock` | `blockHead` () | How confirmed a block must be for empty point-lookups to be retried as missing data. | +| `shortCircuitFutureBlocks` | `*bool` | `nil` = **off**; inherited from `networkDefaults.evm` () | Returns `null` for a numbered `eth_getBlockByNumber` above the highest head any eligible upstream has (static caps included, and syncing upstreams that still receive requests), without dispatching. Always on when `servedTip.enabledFor` includes `latest` (). Does not change what `latest` resolves to. | | `safeBlockSource` | `string` (upstream selector) | `""` = inherit or provider-defined routing | Routes standard block-tagged JSON-RPC requests carrying `safe` to matching upstreams and skips cache reads. The tag stays verbatim so those upstreams define both height and hash. Operator routing overrides client `use-upstream`; retry/consensus still applies inside the matching pool. An empty network value inherits `networkDefaults.evm.safeBlockSource` when present. Does not affect `eth_query*` or direct gRPC Query APIs. | | `evm.integrity.enforceHighestBlock` | `*bool` | `true` () | **Deprecated** — migrated into `directiveDefaults.enforceHighestBlock` at `SetDefaults` time when the directive is unset (). Prefer `directiveDefaults.enforceHighestBlock`. | | `evm.integrity.enforceGetLogsBlockRange` | `*bool` | `true` () | **Deprecated** — same migration path as `enforceHighestBlock`. | @@ -382,7 +385,7 @@ Not present on `SvmNetworkConfig`: there is no `maxSlotsPerSignaturesQuery`. An | `failsafe[]` | `[]FailsafeConfig` | `nil` | Network has none → deep-copied wholesale. Network has some → per-entry merge from the FIRST compatible default (wildcard method + finality match); break on first match (). | | `selectionPolicy` | `SelectionPolicyConfig` | `nil` | Shallow-copied when network's is nil (). | | `directiveDefaults` | `DirectiveDefaultsConfig` | `nil` | Shallow-copied when network's is nil — **no per-field merge** (). | -| `evm` | `EvmNetworkConfig` | `nil` | Struct-copied wholesale when network has no `evm` block — **except onto SVM networks, which never receive it**. Otherwise per-field fill for: `integrity`, `fallbackStatePollerDebounce`, `dynamicBlockTimeDebounceMultiplier`, `blockUnavailableDelayMultiplier`, `fallbackFinalityDepth`, `getLogsMaxAllowed*`, `getLogs*`, `traceFilter*`, `servedTip`, `emptyResultConfidence`. NOT inherited individually: `chainId`, `enforceBlockAvailability`, `maxRetryableBlockDistance`, `markEmptyAsErrorMethods`, `idempotentTransactionBroadcast` (). | +| `evm` | `EvmNetworkConfig` | `nil` | Struct-copied wholesale when network has no `evm` block — **except onto SVM networks, which never receive it**. Otherwise per-field fill for: `integrity`, `fallbackStatePollerDebounce`, `dynamicBlockTimeDebounceMultiplier`, `blockUnavailableDelayMultiplier`, `fallbackFinalityDepth`, `getLogsMaxAllowed*`, `getLogs*`, `traceFilter*`, `servedTip`, `emptyResultConfidence`, `shortCircuitFutureBlocks`. NOT inherited individually: `chainId`, `enforceBlockAvailability`, `maxRetryableBlockDistance`, `markEmptyAsErrorMethods`, `idempotentTransactionBroadcast` (). | | `svm` | `SvmNetworkConfig` | `nil` | Per-field fill: `chain`, `commitment`, `statePollerDebounce`, `maxFinalizedSlotLag`, `enforceBlockAvailability` (). **`cluster` is deliberately NOT inherited** — it is network identity, and inheriting it would let two networks alias one another. The two pointer fields are tested for `nil`, not for their zero value, so an explicit `maxFinalizedSlotLag: 0` or `enforceBlockAvailability: false` set here survives the merge. | | `multiplexing` | `*bool` | `nil` | Value-copied when network's `multiplexing` is nil (). No deep merge — a network must set its own `multiplexing` to deviate from the project-wide default. | @@ -740,14 +743,15 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} 20. **Failed lazy bootstrap is user-retryable** — non-fatal bootstrap tasks are retried by the background auto-retry loop and by the next request's `ExecuteTasks`. Only `resolveNetworkConfig` format errors are `TaskFatal` and never retried. [`erpc/networks_registry.go:L263-266`](https://github.com/erpc/erpc/blob/main/erpc/networks_registry.go#L263-L266) 21. **Provider `onlyNetworks` bypasses vendor support checks entirely** — a networkId listed in `onlyNetworks` returns `SupportsNetwork = true` without consulting the vendor's dynamic check; `ignoreNetworks` is evaluated first and short-circuits to false. [`thirdparty/provider.go:L34-49`](https://github.com/erpc/erpc/blob/main/thirdparty/provider.go#L34-L49) 22. **Synthesized lazy network configs become permanently visible** — `ExposeNetworkConfig` appends to `project.Config.Networks` (admin API surface) and never overwrites an existing entry, so lazy networks persist in the admin response after first use. [`erpc/projects.go:L58-77`](https://github.com/erpc/erpc/blob/main/erpc/projects.go#L58-L77) -23. **Syncing upstreams are excluded from every head computation** — max mode, cluster mode, lowest-finalized, and guaranteed-method floor calculations all skip upstreams in `EvmSyncingStateSyncing`. [`erpc/networks.go:L327-330`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L327-L330) -24. **Future-block short-circuit compares against the MAX eligible head, not the cluster-min served tip** — it never nulls a block that the most-ahead upstream reports having. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L570-610`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L570-L610) +23. **Syncing upstreams are excluded from every head computation** — max mode, cluster mode, lowest-finalized, and guaranteed-method floor calculations all skip upstreams in `EvmSyncingStateSyncing`. The future-block short-circuit is the exception: it checks the heads of syncing upstreams that still receive requests (`evm.skipWhenSyncing` unset or false) separately and dispatches when one of them is at or above the block ([`erpc/networks.go:L638-653`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L638-L653)). [`erpc/networks.go:L327-330`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L327-L330) +24. **Future-block short-circuit compares against the MAX eligible head, not the cluster-min served tip** — it never nulls a block that the most-ahead upstream reports having. It runs only when `evm.shortCircuitFutureBlocks` is true or `servedTip.enabledFor` includes `latest`. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L570-610`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L570-L610) 25. **Wrong-empty misbehavior is suppressed for out-of-bounds blocks** — if the block number in a missing-data response falls outside an upstream's configured `blockAvailability` bounds, the empty response is expected and no misbehavior is recorded against that upstream. [`erpc/networks.go:L1566-1587`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1566-L1587) 26. **`getLogsMaxAllowedRange: 0` is rejected by validation** — the field must be > 0 after defaults (`common/validation.go:L1309-1311`). Unlike `getLogsMaxAllowedAddresses` and `getLogsMaxAllowedTopics` (which default to 0 = unlimited), `getLogsMaxAllowedRange` defaults to 30 000 and cannot be explicitly zeroed out; setting `0` in a network inheriting from defaults is treated as "unset" and inherits the parent value. [`common/defaults.go:L2092-2094`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L2092-L2094) 27. **Zero projects → implicit `main` project + catch-all aliasing rule** — when the config has no projects, eRPC auto-creates `{matchDomain: "*", serveProject: "main"}`, so `/evm/123` works without a project path segment. [`common/defaults.go:L100-110`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L100-L110) 28. **Selector-scoped tips never pollute network gauges** — stateless scoped picks (unmatched or non-simple selectors) use a sentinel lane and emit no Prometheus gauge; equivalent selectors dedup into one partition keyed by matched-set hash; the cap of 16 partitions is enforced globally per network. [`erpc/networks.go:L98-105`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L98-L105) 29. **Static response params matching has no wildcard** — there is no glob or `*` support; every params slot must match exactly. To catch a method regardless of params, add an entry with an empty `params` array. To match block 0 (`"0x0"`), be exact — `"0x00"` will not match. [`common/static_response.go:L14-99`](https://github.com/erpc/erpc/blob/main/common/static_response.go#L14-L99) 30. **Same-chain networks share Redis cache keys unless `cacheKeySuffix` is set** — partition keys are `{networkId}:{blockRef}` (`evm:998:64321354`). Two projects on chainId 998 with a shared Redis will HIT each other's bodies. `X-ERPC-Skip-Cache-Read` still writes, so the last fetch poisons the other project. Set `cacheKeySuffix: systx` on one network to get `evm:998:systx:{blockRef}`. Colon is rejected. Reverse-index wildcard is `{networkId}:{suffix}:*`, passed from the known network id and suffix rather than inferred from the concrete key. Memory/Redis `Delete` must use the same `WithReverseIndex` ctx as `Set`. A chain-wide main-key flush of `evm:998:*` still matches suffixed data keys. [`common/config.go:L3022-L3028`](https://github.com/erpc/erpc/blob/main/common/config.go#L3022-L3028), [`data/connector.go`](https://github.com/erpc/erpc/blob/main/data/connector.go) +31. **`shortCircuitFutureBlocks` can briefly null a block that exists** — the head it compares against is what the state poller and head subscriptions last reported. A block an upstream produced after that gets `null` until eRPC sees it, even if the client already saw it via the node's own `newHeads` (the case covered by `TestNetworkAvailability_Issue934_*`, which run with the option off). Enable it only where heads arrive promptly (WebSocket head subscriptions) and clients retry a `null`. [`erpc/networks.go:L921-952`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L921-L952) ### Observability @@ -780,6 +784,9 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} - `"failed to build static response"` (error) — `NewJsonRpcResponse` failed on a matched static entry. - Provider task: `"registering N upstream(s) from provider"` (`upstream/registry.go`). +**Span attributes** (on the `Network.Forward` span): +- `future_block.short_circuit=true` — the future-block short-circuit returned `null` without dispatching (`evm.shortCircuitFutureBlocks` or served-tip for `latest`). No metric counts these; they are not dispatched, so they do not appear in upstream request metrics. + ### Source code entry points - [`erpc/networks_registry.go:L83-376`](https://github.com/erpc/erpc/blob/main/erpc/networks_registry.go#L83-L376) — `NetworksRegistry`: lazy `GetNetwork`, bootstrap tasks, `resolveNetworkConfig` synthesis, alias map register/resolve, `NewNetwork` constructor. diff --git a/erpc/networks.go b/erpc/networks.go index 678b761b0..85dfe20a3 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -631,6 +631,27 @@ func evmTipObservation(u common.EvmUpstream, useFinalized bool) (blk int64, boun return blk, head > 0 && *exact < head } +// dispatchableSyncingHead returns the highest head among the upstreams that +// report syncing but still receive requests (evm.skipWhenSyncing unset or +// false). evmTipBallot leaves syncing upstreams out of the head reference, so +// the future-block short-circuit checks these separately. +func dispatchableSyncingHead(upstreams []common.Upstream, useFinalized bool) int64 { + var head int64 + for _, cu := range upstreams { + u, ok := cu.(common.EvmUpstream) + if !ok || u.EvmStatePoller() == nil || u.EvmSyncingState() != common.EvmSyncingStateSyncing { + continue + } + if cfg := u.Config(); cfg != nil && cfg.Evm != nil && cfg.Evm.SkipWhenSyncing != nil && *cfg.Evm.SkipWhenSyncing { + continue + } + if blk, _ := evmTipObservation(u, useFinalized); blk > head { + head = blk + } + } + return head +} + // configuredUpperExactBlock returns the upstream's configured // blockAvailability.upper.exactBlock, or nil when it declares no such constant. func configuredUpperExactBlock(u common.Upstream) *int64 { @@ -916,6 +937,11 @@ func (n *Network) tryShortCircuitFutureBlock(ctx context.Context, req *common.No // Unknown head (fail open) or block within reach of some upstream. return nil, false } + if bn <= dispatchableSyncingHead(n.tipCandidateUpstreams(ctx, "*"), useFinalized) { + // Available leaves syncing upstreams out, but one that still receives + // requests may already have the block. + return nil, false + } jrr, err := common.NewJsonRpcResponse(req.ID(), nil, nil) if err != nil { return nil, false diff --git a/erpc/networks_future_block_test.go b/erpc/networks_future_block_test.go index 1a4cac4be..c96868059 100644 --- a/erpc/networks_future_block_test.go +++ b/erpc/networks_future_block_test.go @@ -170,12 +170,48 @@ func TestForward_FutureBlock_LatestTag_Dispatches(t *testing.T) { "latest tag has no concrete future number; must dispatch normally") } +// mockGetBlockByNumberAt makes every listed upstream answer +// eth_getBlockByNumber for blockHex only, counting each dispatch. Matching the +// concrete number keeps the state poller's latest/finalized polls off it, so it +// can be registered before the network is set up. +func mockGetBlockByNumberAt(blockHex, result string, dispatched *atomic.Int64, ids ...string) { + for _, id := range ids { + gock.New("http://" + id + ".localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + if !strings.Contains(body, "eth_getBlockByNumber") || !strings.Contains(body, `"`+blockHex+`"`) { + return false + } + dispatched.Add(1) + return true + }). + Reply(200). + JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":` + result + `}`)) + } +} + +const futureBlockSentinel = `{"number":"0x270f","hash":"0xabc"}` + +func forwardGetBlockByNumber(t *testing.T, ctx context.Context, network *Network, blockHex string) *common.NormalizedResponse { + t.Helper() + req := common.NewNormalizedRequest([]byte( + `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["` + blockHex + `",false]}`)) + resp, err := network.Forward(ctx, req) + require.NoError(t, err) + require.NotNil(t, resp) + return resp +} + // evm.shortCircuitFutureBlocks enables the short-circuit without served-tip: a // block above every head returns null and reaches no upstream. func TestForward_FutureBlock_OptInWithoutServedTip_ShortCircuitsToNull(t *testing.T) { util.ResetGock() defer util.ResetGock() util.SetupMocksForEvmStatePoller() + var dispatched atomic.Int64 + mockGetBlockByNumberAt("0x69", "null", &dispatched, "fb1", "fb2", "fb3") ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -186,27 +222,9 @@ func TestForward_FutureBlock_OptInWithoutServedTip_ShortCircuitsToNull(t *testin {id: "fb3", chainID: 123, latestBlock: 98}, }, nil) network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) - var dispatched atomic.Int64 - for _, id := range []string{"fb1", "fb2", "fb3"} { - gock.New("http://" + id + ".localhost"). - Post(""). - Persist(). - Filter(func(r *http.Request) bool { - if !strings.Contains(util.SafeReadBody(r), "eth_getBlockByNumber") { - return false - } - dispatched.Add(1) - return true - }). - Reply(200). - JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":null}`)) - } - req := common.NewNormalizedRequest([]byte( - `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x69",false]}`)) - resp, err := network.Forward(ctx, req) - require.NoError(t, err) - require.NotNil(t, resp) + // max head = 100; block 105 (0x69) is beyond every upstream. + resp := forwardGetBlockByNumber(t, ctx, network, "0x69") assert.True(t, resp.IsResultEmptyish(ctx), "block 105 > max head 100 must short-circuit to null") assert.Zero(t, dispatched.Load(), @@ -220,6 +238,8 @@ func TestForward_FutureBlock_OptInWithoutServedTip_OnlyMostAheadHasBlock_Dispatc util.ResetGock() defer util.ResetGock() util.SetupMocksForEvmStatePoller() + var dispatched atomic.Int64 + mockGetBlockByNumberAt("0x64", futureBlockSentinel, &dispatched, "fb1", "fb2", "fb3") ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -230,16 +250,70 @@ func TestForward_FutureBlock_OptInWithoutServedTip_OnlyMostAheadHasBlock_Dispatc {id: "fb3", chainID: 123, latestBlock: 98}, }, nil) network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) - mockGetBlockByNumberNonNull("fb1", "fb2", "fb3") // block 100 (0x64) is above the corroborated head (99) but fb1 has it. - req := common.NewNormalizedRequest([]byte( - `{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x64",false]}`)) - resp, err := network.Forward(ctx, req) - require.NoError(t, err) - require.NotNil(t, resp) + resp := forwardGetBlockByNumber(t, ctx, network, "0x64") jrr, err := resp.JsonRpcResponse(ctx) require.NoError(t, err) assert.Contains(t, jrr.GetResultString(), "0x270f", "a block the most-ahead upstream has must be dispatched, not short-circuited") } + +// A syncing upstream is left out of the head reference, but without +// evm.skipWhenSyncing it still receives requests and may have the block, so a +// block at or below its head must be dispatched. +func TestForward_FutureBlock_OptIn_DispatchableSyncingUpstreamAhead_Dispatches(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + var dispatched atomic.Int64 + mockGetBlockByNumberAt("0x69", futureBlockSentinel, &dispatched, "fb1", "fb2", "sync1") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, _ := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "fb1", chainID: 123, latestBlock: 100}, + {id: "fb2", chainID: 123, latestBlock: 99}, + {id: "sync1", chainID: 123, latestBlock: 110, syncing: true}, + }, nil) + network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) + + // block 105 (0x69) is above every non-syncing head (100) but below sync1's. + resp := forwardGetBlockByNumber(t, ctx, network, "0x69") + jrr, err := resp.JsonRpcResponse(ctx) + require.NoError(t, err) + assert.Contains(t, jrr.GetResultString(), "0x270f", + "a dispatchable syncing upstream may have the block; it must be dispatched") +} + +// A syncing upstream with evm.skipWhenSyncing never receives requests, so its +// head does not hold the short-circuit back. +func TestForward_FutureBlock_OptIn_SkippedSyncingUpstreamAhead_ShortCircuitsToNull(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + var dispatched atomic.Int64 + mockGetBlockByNumberAt("0x69", "null", &dispatched, "fb1", "fb2", "sync1") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "fb1", chainID: 123, latestBlock: 100}, + {id: "fb2", chainID: 123, latestBlock: 99}, + {id: "sync1", chainID: 123, latestBlock: 110, syncing: true}, + }, nil) + network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) + for _, u := range ups { + if u.Id() == "sync1" { + u.Config().Evm.SkipWhenSyncing = util.BoolPtr(true) + } + } + + resp := forwardGetBlockByNumber(t, ctx, network, "0x69") + assert.True(t, resp.IsResultEmptyish(ctx), + "a syncing upstream that skips requests cannot serve the block") + assert.Zero(t, dispatched.Load(), + "a block beyond every dispatchable upstream's head must not be dispatched") +} From 5e1c8bdb17e9875988921d83db5fe0351b2d7656 Mon Sep 17 00:00:00 2001 From: shpookas Date: Fri, 9 Oct 2026 13:35:25 +0200 Subject: [PATCH 17/21] fix(evm): compare future blocks against latest heads The future-block short-circuit used finalized heads when emptyResultConfidence is finalizedBlock, so a block between an upstream's finalized and latest head returned null without being dispatched. emptyResultConfidence decides how an empty upstream answer is treated; it should not decide whether an upstream can have the block. Always compare against latest heads, including for dispatchable syncing upstreams. --- docs/pages/config/projects/networks.mdx | 6 ++--- erpc/networks.go | 17 +++++++------ erpc/networks_future_block_test.go | 32 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 29ee4b875..abfdb9b3a 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -334,7 +334,7 @@ https://docs.erpc.cloud/config/projects/networks.llms.txt`} | `idempotentTransactionBroadcast` | `*bool` | `nil` = **enabled** () | "Already known"/"nonce too low"-verified errors become success-with-tx-hash, making retry/hedge safe for `eth_sendRawTransaction`. | | `markEmptyAsErrorMethods` | `[]string` | `eth_blockNumber`, `eth_getBlockByNumber`, `eth_getTransactionByHash`, and 8 others () | Methods where empty response = error (retried, upstream scored down). `eth_getTransactionReceipt` deliberately excluded. | | `emptyResultConfidence` | `blockHead` \| `finalizedBlock` | `blockHead` () | How confirmed a block must be for empty point-lookups to be retried as missing data. | -| `shortCircuitFutureBlocks` | `*bool` | `nil` = **off**; inherited from `networkDefaults.evm` () | Returns `null` for a numbered `eth_getBlockByNumber` above the highest head any eligible upstream has (static caps included, and syncing upstreams that still receive requests), without dispatching. Always on when `servedTip.enabledFor` includes `latest` (). Does not change what `latest` resolves to. | +| `shortCircuitFutureBlocks` | `*bool` | `nil` = **off**; inherited from `networkDefaults.evm` () | Returns `null` for a numbered `eth_getBlockByNumber` above the highest latest head any eligible upstream has (static caps included, and syncing upstreams that still receive requests), without dispatching. Uses latest heads regardless of `emptyResultConfidence`. Always on when `servedTip.enabledFor` includes `latest` (). Does not change what `latest` resolves to. | | `safeBlockSource` | `string` (upstream selector) | `""` = inherit or provider-defined routing | Routes standard block-tagged JSON-RPC requests carrying `safe` to matching upstreams and skips cache reads. The tag stays verbatim so those upstreams define both height and hash. Operator routing overrides client `use-upstream`; retry/consensus still applies inside the matching pool. An empty network value inherits `networkDefaults.evm.safeBlockSource` when present. Does not affect `eth_query*` or direct gRPC Query APIs. | | `evm.integrity.enforceHighestBlock` | `*bool` | `true` () | **Deprecated** — migrated into `directiveDefaults.enforceHighestBlock` at `SetDefaults` time when the directive is unset (). Prefer `directiveDefaults.enforceHighestBlock`. | | `evm.integrity.enforceGetLogsBlockRange` | `*bool` | `true` () | **Deprecated** — same migration path as `enforceHighestBlock`. | @@ -744,14 +744,14 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} 21. **Provider `onlyNetworks` bypasses vendor support checks entirely** — a networkId listed in `onlyNetworks` returns `SupportsNetwork = true` without consulting the vendor's dynamic check; `ignoreNetworks` is evaluated first and short-circuits to false. [`thirdparty/provider.go:L34-49`](https://github.com/erpc/erpc/blob/main/thirdparty/provider.go#L34-L49) 22. **Synthesized lazy network configs become permanently visible** — `ExposeNetworkConfig` appends to `project.Config.Networks` (admin API surface) and never overwrites an existing entry, so lazy networks persist in the admin response after first use. [`erpc/projects.go:L58-77`](https://github.com/erpc/erpc/blob/main/erpc/projects.go#L58-L77) 23. **Syncing upstreams are excluded from every head computation** — max mode, cluster mode, lowest-finalized, and guaranteed-method floor calculations all skip upstreams in `EvmSyncingStateSyncing`. The future-block short-circuit is the exception: it checks the heads of syncing upstreams that still receive requests (`evm.skipWhenSyncing` unset or false) separately and dispatches when one of them is at or above the block ([`erpc/networks.go:L638-653`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L638-L653)). [`erpc/networks.go:L327-330`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L327-L330) -24. **Future-block short-circuit compares against the MAX eligible head, not the cluster-min served tip** — it never nulls a block that the most-ahead upstream reports having. It runs only when `evm.shortCircuitFutureBlocks` is true or `servedTip.enabledFor` includes `latest`. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L570-610`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L570-L610) +24. **Future-block short-circuit compares against the MAX eligible head, not the cluster-min served tip** — it never nulls a block that the most-ahead upstream reports having. It runs only when `evm.shortCircuitFutureBlocks` is true or `servedTip.enabledFor` includes `latest`, and always compares latest heads: `emptyResultConfidence: finalizedBlock` does not make it null an unfinalized block an upstream already has. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L570-610`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L570-L610) 25. **Wrong-empty misbehavior is suppressed for out-of-bounds blocks** — if the block number in a missing-data response falls outside an upstream's configured `blockAvailability` bounds, the empty response is expected and no misbehavior is recorded against that upstream. [`erpc/networks.go:L1566-1587`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1566-L1587) 26. **`getLogsMaxAllowedRange: 0` is rejected by validation** — the field must be > 0 after defaults (`common/validation.go:L1309-1311`). Unlike `getLogsMaxAllowedAddresses` and `getLogsMaxAllowedTopics` (which default to 0 = unlimited), `getLogsMaxAllowedRange` defaults to 30 000 and cannot be explicitly zeroed out; setting `0` in a network inheriting from defaults is treated as "unset" and inherits the parent value. [`common/defaults.go:L2092-2094`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L2092-L2094) 27. **Zero projects → implicit `main` project + catch-all aliasing rule** — when the config has no projects, eRPC auto-creates `{matchDomain: "*", serveProject: "main"}`, so `/evm/123` works without a project path segment. [`common/defaults.go:L100-110`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L100-L110) 28. **Selector-scoped tips never pollute network gauges** — stateless scoped picks (unmatched or non-simple selectors) use a sentinel lane and emit no Prometheus gauge; equivalent selectors dedup into one partition keyed by matched-set hash; the cap of 16 partitions is enforced globally per network. [`erpc/networks.go:L98-105`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L98-L105) 29. **Static response params matching has no wildcard** — there is no glob or `*` support; every params slot must match exactly. To catch a method regardless of params, add an entry with an empty `params` array. To match block 0 (`"0x0"`), be exact — `"0x00"` will not match. [`common/static_response.go:L14-99`](https://github.com/erpc/erpc/blob/main/common/static_response.go#L14-L99) 30. **Same-chain networks share Redis cache keys unless `cacheKeySuffix` is set** — partition keys are `{networkId}:{blockRef}` (`evm:998:64321354`). Two projects on chainId 998 with a shared Redis will HIT each other's bodies. `X-ERPC-Skip-Cache-Read` still writes, so the last fetch poisons the other project. Set `cacheKeySuffix: systx` on one network to get `evm:998:systx:{blockRef}`. Colon is rejected. Reverse-index wildcard is `{networkId}:{suffix}:*`, passed from the known network id and suffix rather than inferred from the concrete key. Memory/Redis `Delete` must use the same `WithReverseIndex` ctx as `Set`. A chain-wide main-key flush of `evm:998:*` still matches suffixed data keys. [`common/config.go:L3022-L3028`](https://github.com/erpc/erpc/blob/main/common/config.go#L3022-L3028), [`data/connector.go`](https://github.com/erpc/erpc/blob/main/data/connector.go) -31. **`shortCircuitFutureBlocks` can briefly null a block that exists** — the head it compares against is what the state poller and head subscriptions last reported. A block an upstream produced after that gets `null` until eRPC sees it, even if the client already saw it via the node's own `newHeads` (the case covered by `TestNetworkAvailability_Issue934_*`, which run with the option off). Enable it only where heads arrive promptly (WebSocket head subscriptions) and clients retry a `null`. [`erpc/networks.go:L921-952`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L921-L952) +31. **`shortCircuitFutureBlocks` can briefly null a block that exists** — the head it compares against is what the state poller and head subscriptions last reported. A block an upstream produced after that gets `null` until eRPC sees it, even if the client already saw it via the node's own `newHeads` (the case covered by `TestNetworkAvailability_Issue934_*`, which run with the option off). Enable it only where heads arrive promptly (WebSocket head subscriptions) and clients retry a `null`. [`erpc/networks.go:L921-955`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L921-L955) ### Observability diff --git a/erpc/networks.go b/erpc/networks.go index 85dfe20a3..41d234707 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -631,11 +631,11 @@ func evmTipObservation(u common.EvmUpstream, useFinalized bool) (blk int64, boun return blk, head > 0 && *exact < head } -// dispatchableSyncingHead returns the highest head among the upstreams that +// dispatchableSyncingHead returns the highest latest head among the upstreams that // report syncing but still receive requests (evm.skipWhenSyncing unset or // false). evmTipBallot leaves syncing upstreams out of the head reference, so // the future-block short-circuit checks these separately. -func dispatchableSyncingHead(upstreams []common.Upstream, useFinalized bool) int64 { +func dispatchableSyncingHead(upstreams []common.Upstream) int64 { var head int64 for _, cu := range upstreams { u, ok := cu.(common.EvmUpstream) @@ -645,7 +645,7 @@ func dispatchableSyncingHead(upstreams []common.Upstream, useFinalized bool) int if cfg := u.Config(); cfg != nil && cfg.Evm != nil && cfg.Evm.SkipWhenSyncing != nil && *cfg.Evm.SkipWhenSyncing { continue } - if blk, _ := evmTipObservation(u, useFinalized); blk > head { + if blk, _ := evmTipObservation(u, false); blk > head { head = blk } } @@ -905,7 +905,7 @@ func (n *Network) evmHeadReference(ctx context.Context, useFinalized bool) serve // tryShortCircuitFutureBlock returns a truthful null response (ok=true) when // `req` is a concrete-numbered eth_getBlockByNumber lookup whose target block is -// beyond every eligible upstream's head (at the network's emptyResultConfidence level). +// beyond every eligible upstream's latest head. // No upstream can serve such a block yet, so dispatching + hedging across all of // them only burns latency and load before they each return empty — returning the // null here skips that fan-out entirely. @@ -931,13 +931,16 @@ func (n *Network) tryShortCircuitFutureBlock(ctx context.Context, req *common.No // params carry no concrete future number — never short-circuit. return nil, false } - useFinalized := n.cfg.Evm.EmptyResultConfidence == common.AvailbilityConfidenceFinalized - maxHead := n.evmHeadReference(ctx, useFinalized).Available + // Latest heads, whatever emptyResultConfidence says: that setting decides how + // an empty upstream answer is treated, while this decides whether any + // upstream can have the block at all. An unfinalized block an upstream + // already has must still be dispatched. + maxHead := n.evmHeadReference(ctx, false).Available if maxHead <= 0 || bn <= maxHead { // Unknown head (fail open) or block within reach of some upstream. return nil, false } - if bn <= dispatchableSyncingHead(n.tipCandidateUpstreams(ctx, "*"), useFinalized) { + if bn <= dispatchableSyncingHead(n.tipCandidateUpstreams(ctx, "*")) { // Available leaves syncing upstreams out, but one that still receives // requests may already have the block. return nil, false diff --git a/erpc/networks_future_block_test.go b/erpc/networks_future_block_test.go index c96868059..67a9b6efd 100644 --- a/erpc/networks_future_block_test.go +++ b/erpc/networks_future_block_test.go @@ -317,3 +317,35 @@ func TestForward_FutureBlock_OptIn_SkippedSyncingUpstreamAhead_ShortCircuitsToNu assert.Zero(t, dispatched.Load(), "a block beyond every dispatchable upstream's head must not be dispatched") } + +// emptyResultConfidence: finalizedBlock decides how an empty upstream answer is +// treated, not whether an upstream can have the block. A block between the +// finalized and latest heads must still be dispatched. +func TestForward_FutureBlock_OptIn_FinalizedConfidence_UnfinalizedBlock_Dispatches(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + var dispatched atomic.Int64 + mockGetBlockByNumberAt("0x5f", futureBlockSentinel, &dispatched, "fb1", "fb2", "fb3") + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + {id: "fb1", chainID: 123, latestBlock: 100}, + {id: "fb2", chainID: 123, latestBlock: 99}, + {id: "fb3", chainID: 123, latestBlock: 98}, + }, nil) + network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) + network.cfg.Evm.EmptyResultConfidence = common.AvailbilityConfidenceFinalized + for _, u := range ups { + u.EvmStatePoller().SuggestFinalizedBlock(90) + } + + // block 95 (0x5f) is above every finalized head (90) but below the latest ones. + resp := forwardGetBlockByNumber(t, ctx, network, "0x5f") + jrr, err := resp.JsonRpcResponse(ctx) + require.NoError(t, err) + assert.Contains(t, jrr.GetResultString(), "0x270f", + "an unfinalized block the upstreams already have must be dispatched") +} From 8d4b27deea47596133b0dc843c2df633d483726c Mon Sep 17 00:00:00 2001 From: shpookas Date: Fri, 9 Oct 2026 14:14:15 +0200 Subject: [PATCH 18/21] fix(evm): bound future blocks by every upstream of the network The future-block ceiling came from the selection policy's wildcard candidates plus a separate pass over syncing upstreams. That missed upstreams a request can still reach: fallbacks the policy cordons off but the per-request escape brings back, and method-scoped policy slots. It also skipped upstreams with an unknown head, so a block one of them already had returned null. Compute the ceiling over every upstream of the network instead, regardless of policy, tier or syncing state, and fail open while any upstream's head is unknown. Replaces dispatchableSyncingHead. --- docs/pages/config/projects/networks.mdx | 10 ++-- erpc/networks.go | 73 ++++++++++++------------- erpc/networks_future_block_test.go | 66 +++++++++++++++------- 3 files changed, 86 insertions(+), 63 deletions(-) diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index abfdb9b3a..f4fd1d454 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -101,7 +101,7 @@ Leave unset for chains where upstreams handle `fromBlock: "0x0"` in an `eth_subs ## Future-block polling -Clients that follow a chain by number (for example rollup nodes reading L1) poll `eth_getBlockByNumber` for the block after the head and treat `null` as "not produced yet". No upstream has that block, so each one returns `null` in turn and every poll walks every routed upstream, fallbacks included. Enable `shortCircuitFutureBlocks` to return `null` for a numbered `eth_getBlockByNumber` above the highest head any eligible upstream has (static availability caps included), without dispatching it. A block any eligible upstream has is still dispatched. It is always on when `servedTip.enabledFor` includes `"latest"`. +Clients that follow a chain by number (for example rollup nodes reading L1) poll `eth_getBlockByNumber` for the block after the head and treat `null` as "not produced yet". No upstream has that block, so each one returns `null` in turn and every poll walks every routed upstream, fallbacks included. Enable `shortCircuitFutureBlocks` to return `null` for a numbered `eth_getBlockByNumber` above the latest head of every upstream of the network (static availability caps included), without dispatching it. A block any upstream has is still dispatched, and the request is dispatched as usual while any upstream's head is unknown. It is always on when `servedTip.enabledFor` includes `"latest"`. @@ -334,7 +334,7 @@ https://docs.erpc.cloud/config/projects/networks.llms.txt`} | `idempotentTransactionBroadcast` | `*bool` | `nil` = **enabled** () | "Already known"/"nonce too low"-verified errors become success-with-tx-hash, making retry/hedge safe for `eth_sendRawTransaction`. | | `markEmptyAsErrorMethods` | `[]string` | `eth_blockNumber`, `eth_getBlockByNumber`, `eth_getTransactionByHash`, and 8 others () | Methods where empty response = error (retried, upstream scored down). `eth_getTransactionReceipt` deliberately excluded. | | `emptyResultConfidence` | `blockHead` \| `finalizedBlock` | `blockHead` () | How confirmed a block must be for empty point-lookups to be retried as missing data. | -| `shortCircuitFutureBlocks` | `*bool` | `nil` = **off**; inherited from `networkDefaults.evm` () | Returns `null` for a numbered `eth_getBlockByNumber` above the highest latest head any eligible upstream has (static caps included, and syncing upstreams that still receive requests), without dispatching. Uses latest heads regardless of `emptyResultConfidence`. Always on when `servedTip.enabledFor` includes `latest` (). Does not change what `latest` resolves to. | +| `shortCircuitFutureBlocks` | `*bool` | `nil` = **off**; inherited from `networkDefaults.evm` () | Returns `null` for a numbered `eth_getBlockByNumber` above the highest latest head across every upstream of the network (static caps included; selection policy, tier and syncing state ignored), without dispatching. Fails open while any upstream's head is unknown. Uses latest heads regardless of `emptyResultConfidence`. Always on when `servedTip.enabledFor` includes `latest` (). Does not change what `latest` resolves to. | | `safeBlockSource` | `string` (upstream selector) | `""` = inherit or provider-defined routing | Routes standard block-tagged JSON-RPC requests carrying `safe` to matching upstreams and skips cache reads. The tag stays verbatim so those upstreams define both height and hash. Operator routing overrides client `use-upstream`; retry/consensus still applies inside the matching pool. An empty network value inherits `networkDefaults.evm.safeBlockSource` when present. Does not affect `eth_query*` or direct gRPC Query APIs. | | `evm.integrity.enforceHighestBlock` | `*bool` | `true` () | **Deprecated** — migrated into `directiveDefaults.enforceHighestBlock` at `SetDefaults` time when the directive is unset (). Prefer `directiveDefaults.enforceHighestBlock`. | | `evm.integrity.enforceGetLogsBlockRange` | `*bool` | `true` () | **Deprecated** — same migration path as `enforceHighestBlock`. | @@ -743,15 +743,15 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} 20. **Failed lazy bootstrap is user-retryable** — non-fatal bootstrap tasks are retried by the background auto-retry loop and by the next request's `ExecuteTasks`. Only `resolveNetworkConfig` format errors are `TaskFatal` and never retried. [`erpc/networks_registry.go:L263-266`](https://github.com/erpc/erpc/blob/main/erpc/networks_registry.go#L263-L266) 21. **Provider `onlyNetworks` bypasses vendor support checks entirely** — a networkId listed in `onlyNetworks` returns `SupportsNetwork = true` without consulting the vendor's dynamic check; `ignoreNetworks` is evaluated first and short-circuits to false. [`thirdparty/provider.go:L34-49`](https://github.com/erpc/erpc/blob/main/thirdparty/provider.go#L34-L49) 22. **Synthesized lazy network configs become permanently visible** — `ExposeNetworkConfig` appends to `project.Config.Networks` (admin API surface) and never overwrites an existing entry, so lazy networks persist in the admin response after first use. [`erpc/projects.go:L58-77`](https://github.com/erpc/erpc/blob/main/erpc/projects.go#L58-L77) -23. **Syncing upstreams are excluded from every head computation** — max mode, cluster mode, lowest-finalized, and guaranteed-method floor calculations all skip upstreams in `EvmSyncingStateSyncing`. The future-block short-circuit is the exception: it checks the heads of syncing upstreams that still receive requests (`evm.skipWhenSyncing` unset or false) separately and dispatches when one of them is at or above the block ([`erpc/networks.go:L638-653`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L638-L653)). [`erpc/networks.go:L327-330`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L327-L330) -24. **Future-block short-circuit compares against the MAX eligible head, not the cluster-min served tip** — it never nulls a block that the most-ahead upstream reports having. It runs only when `evm.shortCircuitFutureBlocks` is true or `servedTip.enabledFor` includes `latest`, and always compares latest heads: `emptyResultConfidence: finalizedBlock` does not make it null an unfinalized block an upstream already has. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L570-610`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L570-L610) +23. **Syncing upstreams are excluded from every head computation** — max mode, cluster mode, lowest-finalized, and guaranteed-method floor calculations all skip upstreams in `EvmSyncingStateSyncing`. The future-block short-circuit is the exception: it counts every upstream of the network, syncing ones included ([`erpc/networks.go:L935-954`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L935-L954)). [`erpc/networks.go:L327-330`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L327-L330) +24. **Future-block short-circuit compares against the MAX head across every upstream of the network, not the cluster-min served tip** — selection policy, tier (a cordoned fallback can still serve through the escape) and syncing state are ignored, and it fails open while any upstream's head is unknown, so it never nulls a block that any upstream reports having. One upstream whose head is never learned (for example a broken endpoint) therefore keeps it from firing on that network. It runs only when `evm.shortCircuitFutureBlocks` is true or `servedTip.enabledFor` includes `latest`, and always compares latest heads: `emptyResultConfidence: finalizedBlock` does not make it null an unfinalized block an upstream already has. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L570-610`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L570-L610) 25. **Wrong-empty misbehavior is suppressed for out-of-bounds blocks** — if the block number in a missing-data response falls outside an upstream's configured `blockAvailability` bounds, the empty response is expected and no misbehavior is recorded against that upstream. [`erpc/networks.go:L1566-1587`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1566-L1587) 26. **`getLogsMaxAllowedRange: 0` is rejected by validation** — the field must be > 0 after defaults (`common/validation.go:L1309-1311`). Unlike `getLogsMaxAllowedAddresses` and `getLogsMaxAllowedTopics` (which default to 0 = unlimited), `getLogsMaxAllowedRange` defaults to 30 000 and cannot be explicitly zeroed out; setting `0` in a network inheriting from defaults is treated as "unset" and inherits the parent value. [`common/defaults.go:L2092-2094`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L2092-L2094) 27. **Zero projects → implicit `main` project + catch-all aliasing rule** — when the config has no projects, eRPC auto-creates `{matchDomain: "*", serveProject: "main"}`, so `/evm/123` works without a project path segment. [`common/defaults.go:L100-110`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L100-L110) 28. **Selector-scoped tips never pollute network gauges** — stateless scoped picks (unmatched or non-simple selectors) use a sentinel lane and emit no Prometheus gauge; equivalent selectors dedup into one partition keyed by matched-set hash; the cap of 16 partitions is enforced globally per network. [`erpc/networks.go:L98-105`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L98-L105) 29. **Static response params matching has no wildcard** — there is no glob or `*` support; every params slot must match exactly. To catch a method regardless of params, add an entry with an empty `params` array. To match block 0 (`"0x0"`), be exact — `"0x00"` will not match. [`common/static_response.go:L14-99`](https://github.com/erpc/erpc/blob/main/common/static_response.go#L14-L99) 30. **Same-chain networks share Redis cache keys unless `cacheKeySuffix` is set** — partition keys are `{networkId}:{blockRef}` (`evm:998:64321354`). Two projects on chainId 998 with a shared Redis will HIT each other's bodies. `X-ERPC-Skip-Cache-Read` still writes, so the last fetch poisons the other project. Set `cacheKeySuffix: systx` on one network to get `evm:998:systx:{blockRef}`. Colon is rejected. Reverse-index wildcard is `{networkId}:{suffix}:*`, passed from the known network id and suffix rather than inferred from the concrete key. Memory/Redis `Delete` must use the same `WithReverseIndex` ctx as `Set`. A chain-wide main-key flush of `evm:998:*` still matches suffixed data keys. [`common/config.go:L3022-L3028`](https://github.com/erpc/erpc/blob/main/common/config.go#L3022-L3028), [`data/connector.go`](https://github.com/erpc/erpc/blob/main/data/connector.go) -31. **`shortCircuitFutureBlocks` can briefly null a block that exists** — the head it compares against is what the state poller and head subscriptions last reported. A block an upstream produced after that gets `null` until eRPC sees it, even if the client already saw it via the node's own `newHeads` (the case covered by `TestNetworkAvailability_Issue934_*`, which run with the option off). Enable it only where heads arrive promptly (WebSocket head subscriptions) and clients retry a `null`. [`erpc/networks.go:L921-955`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L921-L955) +31. **`shortCircuitFutureBlocks` can briefly null a block that exists** — the head it compares against is what the state poller and head subscriptions last reported. A block an upstream produced after that gets `null` until eRPC sees it, even if the client already saw it via the node's own `newHeads` (the case covered by `TestNetworkAvailability_Issue934_*`, which run with the option off). Enable it only where heads arrive promptly (WebSocket head subscriptions) and clients retry a `null`. [`erpc/networks.go:L901-926`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L901-L926) ### Observability diff --git a/erpc/networks.go b/erpc/networks.go index 41d234707..62c25f4d0 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -631,27 +631,6 @@ func evmTipObservation(u common.EvmUpstream, useFinalized bool) (blk int64, boun return blk, head > 0 && *exact < head } -// dispatchableSyncingHead returns the highest latest head among the upstreams that -// report syncing but still receive requests (evm.skipWhenSyncing unset or -// false). evmTipBallot leaves syncing upstreams out of the head reference, so -// the future-block short-circuit checks these separately. -func dispatchableSyncingHead(upstreams []common.Upstream) int64 { - var head int64 - for _, cu := range upstreams { - u, ok := cu.(common.EvmUpstream) - if !ok || u.EvmStatePoller() == nil || u.EvmSyncingState() != common.EvmSyncingStateSyncing { - continue - } - if cfg := u.Config(); cfg != nil && cfg.Evm != nil && cfg.Evm.SkipWhenSyncing != nil && *cfg.Evm.SkipWhenSyncing { - continue - } - if blk, _ := evmTipObservation(u, false); blk > head { - head = blk - } - } - return head -} - // configuredUpperExactBlock returns the upstream's configured // blockAvailability.upper.exactBlock, or nil when it declares no such constant. func configuredUpperExactBlock(u common.Upstream) *int64 { @@ -905,14 +884,15 @@ func (n *Network) evmHeadReference(ctx context.Context, useFinalized bool) serve // tryShortCircuitFutureBlock returns a truthful null response (ok=true) when // `req` is a concrete-numbered eth_getBlockByNumber lookup whose target block is -// beyond every eligible upstream's latest head. +// beyond every upstream's latest head. // No upstream can serve such a block yet, so dispatching + hedging across all of // them only burns latency and load before they each return empty — returning the // null here skips that fan-out entirely. // -// Safety: it compares against the highest effective head across eligible -// upstreams, static caps included (servedTipReference.Available), so it never -// nulls out a block any upstream actually serves. It runs when the network sets +// Safety: it compares against the highest effective latest head across every +// upstream of the network, static caps included, and fails open when any of +// them has no known head (see futureBlockCeiling), so it never nulls out a +// block any upstream actually serves. It runs when the network sets // evm.shortCircuitFutureBlocks or enables served-tip for the latest axis (see // EvmNetworkConfig.ShortCircuitFutureBlocksEnabled), and the synthesized // response is returned directly from Forward, so it is never written to cache @@ -931,20 +911,11 @@ func (n *Network) tryShortCircuitFutureBlock(ctx context.Context, req *common.No // params carry no concrete future number — never short-circuit. return nil, false } - // Latest heads, whatever emptyResultConfidence says: that setting decides how - // an empty upstream answer is treated, while this decides whether any - // upstream can have the block at all. An unfinalized block an upstream - // already has must still be dispatched. - maxHead := n.evmHeadReference(ctx, false).Available - if maxHead <= 0 || bn <= maxHead { + maxHead, known := n.futureBlockCeiling(ctx) + if !known || bn <= maxHead { // Unknown head (fail open) or block within reach of some upstream. return nil, false } - if bn <= dispatchableSyncingHead(n.tipCandidateUpstreams(ctx, "*")) { - // Available leaves syncing upstreams out, but one that still receives - // requests may already have the block. - return nil, false - } jrr, err := common.NewJsonRpcResponse(req.ID(), nil, nil) if err != nil { return nil, false @@ -954,6 +925,34 @@ func (n *Network) tryShortCircuitFutureBlock(ctx context.Context, req *common.No return resp, true } +// futureBlockCeiling returns the highest effective latest head across every +// upstream of the network, regardless of selection policy, tier or syncing +// state: any of them can end up serving a request (policy order, the fallback +// escape, syncing upstreams without skipWhenSyncing). It uses latest heads +// whatever emptyResultConfidence says: that setting decides how an empty answer +// is treated, not whether an upstream can have the block. known is false when +// the network has no upstreams or any upstream's head is unknown. +func (n *Network) futureBlockCeiling(ctx context.Context) (head int64, known bool) { + if n.upstreamsRegistry == nil { + return 0, false + } + ups := n.upstreamsRegistry.GetNetworkUpstreams(ctx, n.networkId) + if len(ups) == 0 { + return 0, false + } + for _, u := range ups { + if u.EvmStatePoller() == nil { + return 0, false + } + blk, _ := evmTipObservation(u, false) + if blk <= 0 { + return 0, false + } + head = max(head, blk) + } + return head, true +} + // servedTip computes the majority served tip for one axis over the // selection-policy-eligible upstreams — the freshest block a strict majority of // them already has (see common.PickServedTip) — lets the trajectory referee prefer @@ -1513,7 +1512,7 @@ func (n *Network) SvmEnforceBlockAvailability() bool { // This is used only by the networkPreForward_getBlock guard — its job is to // avoid short-circuiting requests that ANY upstream can serve. It therefore // uses MAX, not the majority (median) tip. This mirrors how EVM's -// tryShortCircuitFutureBlock uses evmHeadReference.Max (not PickServedTip.Tip): +// tryShortCircuitFutureBlock uses the max head across upstreams (not PickServedTip.Tip): // "never null out a block the most-ahead upstream actually has." // SvmHighestLatestSlot / SvmHighestFinalizedSlot remain median-based because // they advertise the chain head to clients — a different, conservative goal. diff --git a/erpc/networks_future_block_test.go b/erpc/networks_future_block_test.go index 67a9b6efd..5aeeb5337 100644 --- a/erpc/networks_future_block_test.go +++ b/erpc/networks_future_block_test.go @@ -259,10 +259,9 @@ func TestForward_FutureBlock_OptInWithoutServedTip_OnlyMostAheadHasBlock_Dispatc "a block the most-ahead upstream has must be dispatched, not short-circuited") } -// A syncing upstream is left out of the head reference, but without -// evm.skipWhenSyncing it still receives requests and may have the block, so a -// block at or below its head must be dispatched. -func TestForward_FutureBlock_OptIn_DispatchableSyncingUpstreamAhead_Dispatches(t *testing.T) { +// Every upstream of the network counts toward the ceiling, syncing ones +// included: a block at or below a syncing upstream's head must be dispatched. +func TestForward_FutureBlock_OptIn_SyncingUpstreamAhead_Dispatches(t *testing.T) { util.ResetGock() defer util.ResetGock() util.SetupMocksForEvmStatePoller() @@ -284,38 +283,36 @@ func TestForward_FutureBlock_OptIn_DispatchableSyncingUpstreamAhead_Dispatches(t jrr, err := resp.JsonRpcResponse(ctx) require.NoError(t, err) assert.Contains(t, jrr.GetResultString(), "0x270f", - "a dispatchable syncing upstream may have the block; it must be dispatched") + "a syncing upstream may have the block; it must be dispatched") } -// A syncing upstream with evm.skipWhenSyncing never receives requests, so its -// head does not hold the short-circuit back. -func TestForward_FutureBlock_OptIn_SkippedSyncingUpstreamAhead_ShortCircuitsToNull(t *testing.T) { +// An upstream with no known head may already have the block, so the +// short-circuit fails open and the request is dispatched. +func TestForward_FutureBlock_OptIn_UpstreamWithUnknownHead_Dispatches(t *testing.T) { util.ResetGock() defer util.ResetGock() util.SetupMocksForEvmStatePoller() var dispatched atomic.Int64 - mockGetBlockByNumberAt("0x69", "null", &dispatched, "fb1", "fb2", "sync1") + mockGetBlockByNumberAt("0x65", futureBlockSentinel, &dispatched, "fb1", "fb2", "nohead") ctx, cancel := context.WithCancel(context.Background()) defer cancel() - network, ups := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ + network, _ := setupServedTipNetworkWith(t, ctx, []servedTipFixture{ {id: "fb1", chainID: 123, latestBlock: 100}, {id: "fb2", chainID: 123, latestBlock: 99}, - {id: "sync1", chainID: 123, latestBlock: 110, syncing: true}, + // no latestBlock: the poller never learns a head for this upstream + {id: "nohead", chainID: 123}, }, nil) network.cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) - for _, u := range ups { - if u.Id() == "sync1" { - u.Config().Evm.SkipWhenSyncing = util.BoolPtr(true) - } - } - resp := forwardGetBlockByNumber(t, ctx, network, "0x69") - assert.True(t, resp.IsResultEmptyish(ctx), - "a syncing upstream that skips requests cannot serve the block") - assert.Zero(t, dispatched.Load(), - "a block beyond every dispatchable upstream's head must not be dispatched") + // block 101 (0x65) is above every known head (100). + resp := forwardGetBlockByNumber(t, ctx, network, "0x65") + jrr, err := resp.JsonRpcResponse(ctx) + require.NoError(t, err) + assert.Contains(t, jrr.GetResultString(), "0x270f", + "an upstream with an unknown head may have the block; it must be dispatched") + assert.Positive(t, dispatched.Load()) } // emptyResultConfidence: finalizedBlock decides how an empty upstream answer is @@ -349,3 +346,30 @@ func TestForward_FutureBlock_OptIn_FinalizedConfidence_UnfinalizedBlock_Dispatch assert.Contains(t, jrr.GetResultString(), "0x270f", "an unfinalized block the upstreams already have must be dispatched") } + +// A fallback the selection policy cordons off can still serve through the +// per-request escape, so its head counts toward the ceiling: a block between +// the primaries' head and the fallbacks' head is dispatched, not nulled. +func TestForward_FutureBlock_OptIn_CordonedFallbackAhead_Dispatches(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer util.ResetGock() + + var dispatched atomic.Int64 + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x64", // 100 + fallbackLatest: "0x6e", // 110 + enableFailover: true, + mocks: func() { + mockGetBlockByNumberAt("0x69", "null", &dispatched, "rpc1", "rpc2", "rpc3", "rpc4") + }, + network: func(cfg *common.NetworkConfig) { + cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) + }, + }) + + // block 105 (0x69) is above the primaries' head but below the fallbacks'. + forwardGetBlockByNumber(t, ctx, network, "0x69") + assert.Positive(t, dispatched.Load(), + "a block a cordoned fallback may have must be dispatched, not short-circuited") +} From decea44f30d9438cb9f0f007a4e5bb4480cb326c Mon Sep 17 00:00:00 2001 From: shpookas Date: Fri, 9 Oct 2026 15:21:20 +0200 Subject: [PATCH 19/21] fix(network): run the future-block short-circuit before the probe bus Forward published each request to the selection policy's probe bus before the future-block short-circuit, so probeExcluded still mirrored requests that were answered with a synthetic null to excluded upstreams. Run the short-circuit first. It only matches eth_getBlockByNumber, which the network pre-forward hook does not handle, so nothing else moves. Also describe the ceiling as every upstream of the network in the config comment, TypeScript type and docs, and point edge case 24 at the short-circuit code. --- common/config.go | 6 +++-- docs/pages/config/projects/networks.mdx | 6 ++--- erpc/networks.go | 29 +++++++++++----------- erpc/networks_future_block_test.go | 32 +++++++++++++++++++++++++ typescript/config/src/generated.ts | 4 +++- 5 files changed, 57 insertions(+), 20 deletions(-) diff --git a/common/config.go b/common/config.go index 226b4f46f..ab99240bd 100644 --- a/common/config.go +++ b/common/config.go @@ -2682,7 +2682,9 @@ type EvmNetworkConfig struct { EmptyResultConfidence AvailbilityConfidence `yaml:"emptyResultConfidence,omitempty" json:"emptyResultConfidence,omitempty"` // ShortCircuitFutureBlocks returns null for a numbered eth_getBlockByNumber - // above the highest head any eligible upstream has, without dispatching it. + // above the highest latest head across every upstream of the network + // (regardless of selection policy, tier or syncing state), without + // dispatching it; it never fires while any upstream's head is unknown. // Clients that poll the next block by number otherwise walk every routed // upstream on each poll. Always on when servedTip is enabled for "latest". // The head is what the state poller and head subscriptions last reported, so @@ -2771,7 +2773,7 @@ type EvmServedTipConfig struct { } // ShortCircuitFutureBlocksEnabled reports whether a numbered eth_getBlockByNumber -// above every eligible upstream's head returns null without dispatching: when +// above every upstream's head returns null without dispatching: when // ShortCircuitFutureBlocks is true, or servedTip is enabled for "latest". // Nil-receiver safe. func (c *EvmNetworkConfig) ShortCircuitFutureBlocksEnabled() bool { diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index f4fd1d454..1d21474cf 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -113,7 +113,7 @@ projects: evm: chainId: 1 # Returns null for eth_getBlockByNumber(N) when N is above every - # eligible upstream's head, instead of asking each upstream. + # upstream's head, instead of asking each upstream. # default: false shortCircuitFutureBlocks: true ``` @@ -132,7 +132,7 @@ export default createConfig({ evm: { chainId: 1, // Returns null for eth_getBlockByNumber(N) when N is above every - // eligible upstream's head, instead of asking each upstream. + // upstream's head, instead of asking each upstream. // default: false shortCircuitFutureBlocks: true, }, @@ -744,7 +744,7 @@ POST /main → body {"networkId": "svm:fogo:mainnet", ...} 21. **Provider `onlyNetworks` bypasses vendor support checks entirely** — a networkId listed in `onlyNetworks` returns `SupportsNetwork = true` without consulting the vendor's dynamic check; `ignoreNetworks` is evaluated first and short-circuits to false. [`thirdparty/provider.go:L34-49`](https://github.com/erpc/erpc/blob/main/thirdparty/provider.go#L34-L49) 22. **Synthesized lazy network configs become permanently visible** — `ExposeNetworkConfig` appends to `project.Config.Networks` (admin API surface) and never overwrites an existing entry, so lazy networks persist in the admin response after first use. [`erpc/projects.go:L58-77`](https://github.com/erpc/erpc/blob/main/erpc/projects.go#L58-L77) 23. **Syncing upstreams are excluded from every head computation** — max mode, cluster mode, lowest-finalized, and guaranteed-method floor calculations all skip upstreams in `EvmSyncingStateSyncing`. The future-block short-circuit is the exception: it counts every upstream of the network, syncing ones included ([`erpc/networks.go:L935-954`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L935-L954)). [`erpc/networks.go:L327-330`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L327-L330) -24. **Future-block short-circuit compares against the MAX head across every upstream of the network, not the cluster-min served tip** — selection policy, tier (a cordoned fallback can still serve through the escape) and syncing state are ignored, and it fails open while any upstream's head is unknown, so it never nulls a block that any upstream reports having. One upstream whose head is never learned (for example a broken endpoint) therefore keeps it from firing on that network. It runs only when `evm.shortCircuitFutureBlocks` is true or `servedTip.enabledFor` includes `latest`, and always compares latest heads: `emptyResultConfidence: finalizedBlock` does not make it null an unfinalized block an upstream already has. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L570-610`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L570-L610) +24. **Future-block short-circuit compares against the MAX head across every upstream of the network, not the cluster-min served tip** — selection policy, tier (a cordoned fallback can still serve through the escape) and syncing state are ignored, and it fails open while any upstream's head is unknown, so it never nulls a block that any upstream reports having. One upstream whose head is never learned (for example a broken endpoint) therefore keeps it from firing on that network. It runs only when `evm.shortCircuitFutureBlocks` is true or `servedTip.enabledFor` includes `latest`, and always compares latest heads: `emptyResultConfidence: finalizedBlock` does not make it null an unfinalized block an upstream already has. The synthetic null response is never cached, and the short-circuit only fires for concrete block numbers (not tags or hashes). [`erpc/networks.go:L901-954`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L901-L954) 25. **Wrong-empty misbehavior is suppressed for out-of-bounds blocks** — if the block number in a missing-data response falls outside an upstream's configured `blockAvailability` bounds, the empty response is expected and no misbehavior is recorded against that upstream. [`erpc/networks.go:L1566-1587`](https://github.com/erpc/erpc/blob/main/erpc/networks.go#L1566-L1587) 26. **`getLogsMaxAllowedRange: 0` is rejected by validation** — the field must be > 0 after defaults (`common/validation.go:L1309-1311`). Unlike `getLogsMaxAllowedAddresses` and `getLogsMaxAllowedTopics` (which default to 0 = unlimited), `getLogsMaxAllowedRange` defaults to 30 000 and cannot be explicitly zeroed out; setting `0` in a network inheriting from defaults is treated as "unset" and inherits the parent value. [`common/defaults.go:L2092-2094`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L2092-L2094) 27. **Zero projects → implicit `main` project + catch-all aliasing rule** — when the config has no projects, eRPC auto-creates `{matchDomain: "*", serveProject: "main"}`, so `/evm/123` works without a project path segment. [`common/defaults.go:L100-110`](https://github.com/erpc/erpc/blob/main/common/defaults.go#L100-L110) diff --git a/erpc/networks.go b/erpc/networks.go index 62c25f4d0..d7ce68bac 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -2160,10 +2160,23 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* // Set upstreams on the request req.SetUpstreams(upsList) + // Future-block short-circuit: a concrete block number beyond every + // upstream's head cannot be served yet — return the truthful null instead of + // dispatching + hedging across upstreams that will all return empty. Runs + // before rate limiting so a non-dispatched request consumes no permit, and + // before the probe-bus publish so it is never mirrored to excluded upstreams. + if resp, ok := n.tryShortCircuitFutureBlock(ctx, req, method); ok { + forwardSpan.SetAttributes(attribute.Bool("future_block.short_circuit", true)) + if mlx != nil { + mlx.Close(ctx, resp, nil) + } + return resp, nil + } + // Feed the per-network probe-bus AFTER we know the request is // actually going to dispatch to an upstream (i.e. not a - // cache-hit / static-response / follower-multiplexer - // short-circuit, all of which returned earlier). The publish is + // cache-hit / static-response / follower-multiplexer / + // future-block short-circuit, all of which returned earlier). The publish is // non-blocking and drops on overflow — request latency is never // affected. The Prober (if any) samples from this feed to mirror // the request against currently-excluded upstreams so their @@ -2190,18 +2203,6 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* } } - // Future-block short-circuit: a concrete block number beyond every eligible - // upstream's head cannot be served yet — return the truthful null instead of - // dispatching + hedging across upstreams that will all return empty. Runs - // before rate limiting so a non-dispatched request consumes no permit. - if resp, ok := n.tryShortCircuitFutureBlock(ctx, req, method); ok { - forwardSpan.SetAttributes(attribute.Bool("future_block.short_circuit", true)) - if mlx != nil { - mlx.Close(ctx, resp, nil) - } - return resp, nil - } - // 3) Check if we should handle this method on this network if err := n.shouldHandleMethod(req, method, upsList); err != nil { if mlx != nil { diff --git a/erpc/networks_future_block_test.go b/erpc/networks_future_block_test.go index 5aeeb5337..db5367630 100644 --- a/erpc/networks_future_block_test.go +++ b/erpc/networks_future_block_test.go @@ -6,6 +6,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/erpc/erpc/common" "github.com/erpc/erpc/util" @@ -373,3 +374,34 @@ func TestForward_FutureBlock_OptIn_CordonedFallbackAhead_Dispatches(t *testing.T assert.Positive(t, dispatched.Load(), "a block a cordoned fallback may have must be dispatched, not short-circuited") } + +// The default selection policy mirrors sampled requests to the upstreams it +// excludes (probeExcluded). A short-circuited request is never dispatched, so it +// must not be mirrored either. +func TestForward_FutureBlock_OptIn_ShortCircuitNotMirroredToExcluded(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer util.ResetGock() + + var dispatched atomic.Int64 + network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ + primaryLatest: "0x64", // 100 + fallbackLatest: "0x64", // 100, cordoned by the default policy + enableFailover: true, + mocks: func() { + mockGetBlockByNumberAt("0x69", "null", &dispatched, "rpc1", "rpc2", "rpc3", "rpc4") + }, + network: func(cfg *common.NetworkConfig) { + cfg.Evm.ShortCircuitFutureBlocks = util.BoolPtr(true) + }, + }) + + for i := 0; i < 5; i++ { + resp := forwardGetBlockByNumber(t, ctx, network, "0x69") + assert.True(t, resp.IsResultEmptyish(ctx)) + } + // Mirrors are asynchronous; give the prober time to dispatch any. + time.Sleep(500 * time.Millisecond) + assert.Zero(t, dispatched.Load(), + "a short-circuited request must reach no upstream, mirrored probes included") +} diff --git a/typescript/config/src/generated.ts b/typescript/config/src/generated.ts index f0d773d04..62d746583 100644 --- a/typescript/config/src/generated.ts +++ b/typescript/config/src/generated.ts @@ -1704,7 +1704,9 @@ export interface EvmNetworkConfig { emptyResultConfidence?: AvailbilityConfidence; /** * ShortCircuitFutureBlocks returns null for a numbered eth_getBlockByNumber - * above the highest head any eligible upstream has, without dispatching it. + * above the highest latest head across every upstream of the network + * (regardless of selection policy, tier or syncing state), without + * dispatching it; it never fires while any upstream's head is unknown. * Clients that poll the next block by number otherwise walk every routed * upstream on each poll. Always on when servedTip is enabled for "latest". * The head is what the state poller and head subscriptions last reported, so From dbecbcb9828c16541e0b89c67e8ee651c03a7888 Mon Sep 17 00:00:00 2001 From: shpookas Date: Fri, 9 Oct 2026 15:45:35 +0200 Subject: [PATCH 20/21] docs(evm): state the future-block guarantee in terms of observed heads The short-circuit compares against the last heads eRPC observed, so it never nulls a block at or below a reported head, but a block produced after the last observed head can still get null. Say that in the tryShortCircuitFutureBlock comment and the docs instead of promising any block an upstream has is dispatched. --- docs/pages/config/projects/networks.mdx | 2 +- erpc/networks.go | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 1d21474cf..120217f44 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -101,7 +101,7 @@ Leave unset for chains where upstreams handle `fromBlock: "0x0"` in an `eth_subs ## Future-block polling -Clients that follow a chain by number (for example rollup nodes reading L1) poll `eth_getBlockByNumber` for the block after the head and treat `null` as "not produced yet". No upstream has that block, so each one returns `null` in turn and every poll walks every routed upstream, fallbacks included. Enable `shortCircuitFutureBlocks` to return `null` for a numbered `eth_getBlockByNumber` above the latest head of every upstream of the network (static availability caps included), without dispatching it. A block any upstream has is still dispatched, and the request is dispatched as usual while any upstream's head is unknown. It is always on when `servedTip.enabledFor` includes `"latest"`. +Clients that follow a chain by number (for example rollup nodes reading L1) poll `eth_getBlockByNumber` for the block after the head and treat `null` as "not produced yet". No upstream has that block, so each one returns `null` in turn and every poll walks every routed upstream, fallbacks included. Enable `shortCircuitFutureBlocks` to return `null` for a numbered `eth_getBlockByNumber` above the latest head of every upstream of the network (static availability caps included), without dispatching it. A block at or below the last head any upstream reported is still dispatched, and the request is dispatched as usual while any upstream's head is unknown. It is always on when `servedTip.enabledFor` includes `"latest"`. diff --git a/erpc/networks.go b/erpc/networks.go index d7ce68bac..f24482220 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -889,10 +889,12 @@ func (n *Network) evmHeadReference(ctx context.Context, useFinalized bool) serve // them only burns latency and load before they each return empty — returning the // null here skips that fan-out entirely. // -// Safety: it compares against the highest effective latest head across every -// upstream of the network, static caps included, and fails open when any of -// them has no known head (see futureBlockCeiling), so it never nulls out a -// block any upstream actually serves. It runs when the network sets +// Safety: it compares against the highest last-observed effective latest head +// across every upstream of the network, static caps included, and fails open +// when any of them has no observed head (see futureBlockCeiling), so it never +// nulls out a block at or below a head an upstream has reported. A block an +// upstream produced after its last observed head can still get null until the +// poller or a head subscription reports it. It runs when the network sets // evm.shortCircuitFutureBlocks or enables served-tip for the latest axis (see // EvmNetworkConfig.ShortCircuitFutureBlocksEnabled), and the synthesized // response is returned directly from Forward, so it is never written to cache From 01dc553e4800be6a4cbe59257fa39f558506caef Mon Sep 17 00:00:00 2001 From: shpookas Date: Mon, 12 Oct 2026 11:53:32 +0200 Subject: [PATCH 21/21] refactor(network): drop servedTipReference.Available The future-block short-circuit now uses futureBlockCeiling, so nothing reads Available. Remove it and update the evmHeadReference comment. --- erpc/networks.go | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/erpc/networks.go b/erpc/networks.go index 72318360e..2cafd468f 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -513,7 +513,6 @@ func (n *Network) gatherEvmTipInputsForMethod( func evmTipBallot(upstreams []common.Upstream, useFinalized bool) (common.ServedTipPick, servedTipReference) { live := make([]common.ServedTipInput, 0, len(upstreams)) var capped []common.ServedTipInput - var available int64 for _, cu := range upstreams { u, ok := cu.(common.EvmUpstream) if !ok || u.EvmStatePoller() == nil { @@ -526,7 +525,6 @@ func evmTipBallot(upstreams []common.Upstream, useFinalized bool) (common.Served if blk <= 0 { continue } - available = max(available, blk) in := common.ServedTipInput{ UpstreamID: u.Id(), BlockNumber: blk, @@ -542,10 +540,10 @@ func evmTipBallot(upstreams []common.Upstream, useFinalized bool) (common.Served // the only observations available. (No live head means no live reference // for the regression guard either; it falls back to its own last // corroborated pick, see guardServedTipRegression.) - return common.PickServedTip(capped), servedTipReference{Available: available} + return common.PickServedTip(capped), servedTipReference{} } pick := common.PickServedTip(live) - return pick, servedTipReference{Corroborated: pick.Freshest, Max: pick.Max, Available: available} + return pick, servedTipReference{Corroborated: pick.Freshest, Max: pick.Max} } // servedTipReference is what the LIVE (non-capped) heads of one ballot say, @@ -582,12 +580,6 @@ type servedTipReference struct { // pick in the very shape the guard exists for (one live witness left, two // stuck upstreams outvoting it). Max int64 - - // Available is the highest effective head across EVERY candidate, live or - // static-capped: the bound above which no candidate holds the block. Only - // the future-block short-circuit reads it; the ballot and the guard ignore - // caps, but a capped archive still serves the blocks below its cap. - Available int64 } // evmTipObservation returns an upstream's effective head for the axis, plus @@ -872,12 +864,12 @@ func (n *Network) servedTipEnabledFor(tag string) bool { return n.cfg != nil && n.cfg.Evm != nil && n.cfg.Evm.ServedTipEnabledFor(tag) } -// evmHeadReference is the corroborated head, the raw live max and the servable -// ceiling over the eligible, non-syncing upstreams for one axis — the same -// ballot the served tip votes on. It serves the default (non-served-tip) head -// accessors and the future-block short-circuit. A cap-only set has no live -// reference, so its caps stand in: they are the only heads anyone in it can -// serve. +// evmHeadReference is the corroborated head and the raw live max over the +// eligible, non-syncing upstreams for one axis — the same ballot the served tip +// votes on. It serves the default (non-served-tip) head accessors; the +// future-block short-circuit uses futureBlockCeiling instead. A cap-only set has +// no live reference, so its caps stand in: they are the only heads anyone in it +// can serve. func (n *Network) evmHeadReference(ctx context.Context, useFinalized bool) servedTipReference { pick, ref := evmTipBallot(n.tipCandidateUpstreams(ctx, "*"), useFinalized) if ref.Max <= 0 && pick.Inputs > 0 {