From b1e866ef89e4ab4dbca6cfc002b40b026b5193a1 Mon Sep 17 00:00:00 2001 From: Bolek <1416262+bolekk@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:01:51 +0000 Subject: [PATCH 1/8] [CRE-6796] Migrate WebAPI and Dummy gateway handlers to sharded DONs config (#23830) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Migrate Dummy and WebAPI (v0.1) gateway handlers to sharded DONs Both handlers previously only worked against a single DON shard, because handler_factory.go collapsed the full shardedDONs/shardsConnMgrs matrix down to shard 0 of DON 0 before constructing them. Add a shared handlers.NodeRouter that expands a DON×shard matrix into a node address -> connection manager lookup, and use it in: - DummyHandler: fan out requests to every node across every DON/shard. - WebAPI (v0.1 web-api-capabilities) handler: route the HTTP fetch response back through the connection manager that actually owns the requesting node, instead of a single fixed one. handler_factory.go now passes shardedDONs/shardsConnMgrs straight through for these two handler types, matching the HTTP capability handler. Vault and ConfidentialRelay still use the legacy single-shard conversion; see z_tracker/projects/gateway_handler_sharding.md for the follow-up plan (CRE-1640). Co-Authored-By: Claude Sonnet 5 * Bundle shardedDONs/shardsConnMgrs into a ShardedDONs struct DummyHandler and the WebAPI (v0.1) handler threaded shardedDONs and shardsConnMgrs as two parallel slices that must always agree in length, re-validating that invariant independently in handler_factory.go and BuildNodeRouter. Introduce handlers.ShardedDONs to pair them once, via NewShardedDONs, and give it a BuildNodeRouter method so handler constructors take a single validated value instead of two slices. node_router.go is renamed to shard_endpoints.go to hold it, mirroring the naming already used for the analogous matrix-builder in capabilities/v2. Have the HTTP capability handler take *handlers.ShardedDONs too NewGatewayHandler (v1.0 HTTP capabilities) still took shardedDONs and shardsConnMgrs as two parallel slices, so its own buildShardEndpoints re-validated the same length/nil invariants that ShardedDONs already checks. Switch it to accept *handlers.ShardedDONs like DummyHandler and the WebAPI (v0.1) handler now do, and drop the now-redundant validation from buildShardEndpoints, which only needs to check that shard memberships are disjoint. Unify NodeRouter and v2's shardEndpoint into one ShardEndpoint type We had two structs doing the same DON×shard matrix expansion: handlers.NodeRouter (a flat nodeAddr -> connMgr map, used by DummyHandler and the WebAPI v0.1 handler) and v2's private shardEndpoint (which also kept the shard's donID/F, used by the HTTP v1.0 capability handler and its trigger/metadata sub-handlers). Replace both with a single exported handlers.ShardEndpoint plus a (*ShardedDONs).BuildShardEndpoints method in handlers/shard_endpoints.go, and point every handler (Dummy, WebAPI v0.1, HTTP v1.0) at it. Delete v2/shard_endpoints.go now that its logic lives in one place. Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Claude Sonnet 5 --- core/services/gateway/handler_factory.go | 31 ++-- .../gateway/handlers/capabilities/handler.go | 27 +++- .../handlers/capabilities/handler_test.go | 18 ++- .../handlers/capabilities/v2/http_handler.go | 15 +- .../capabilities/v2/http_handler_test.go | 79 +++++----- .../capabilities/v2/http_trigger_handler.go | 40 ++--- .../v2/http_trigger_handler_test.go | 60 ++++--- .../capabilities/v2/outbound_bounds_test.go | 6 +- .../capabilities/v2/shard_endpoints.go | 77 --------- .../v2/workflow_metadata_handler.go | 39 ++--- .../v2/workflow_metadata_handler_test.go | 54 ++++--- .../gateway/handlers/handler.dummy.go | 33 ++-- .../gateway/handlers/handler.dummy_test.go | 19 ++- .../gateway/handlers/shard_endpoints.go | 99 ++++++++++++ .../v2 => }/shard_endpoints_test.go | 148 ++++++++++-------- 15 files changed, 421 insertions(+), 324 deletions(-) delete mode 100644 core/services/gateway/handlers/capabilities/v2/shard_endpoints.go create mode 100644 core/services/gateway/handlers/shard_endpoints.go rename core/services/gateway/handlers/{capabilities/v2 => }/shard_endpoints_test.go (58%) diff --git a/core/services/gateway/handler_factory.go b/core/services/gateway/handler_factory.go index a3755c16f6b..0034efa691a 100644 --- a/core/services/gateway/handler_factory.go +++ b/core/services/gateway/handler_factory.go @@ -2,7 +2,6 @@ package gateway import ( "encoding/json" - "errors" "fmt" "github.com/jonboulle/clockwork" @@ -65,29 +64,31 @@ func (hf *handlerFactory) NewHandler( shardedDONs []config.ShardedDONConfig, shardsConnMgrs [][]handlers.DON, ) (handlers.Handler, error) { - if len(shardedDONs) == 0 || len(shardsConnMgrs) == 0 { - return nil, errors.New("at least one DON and connection manager required") + dons, err := handlers.NewShardedDONs(shardedDONs, shardsConnMgrs) + if err != nil { + return nil, err } - if len(shardsConnMgrs[0]) == 0 { - return nil, errors.New("at least one shard connection manager required") - } - - // For backward compatibility, convert sharded config to legacy DONConfig - // using the first DON's first shard. Handlers that support sharding can - // use the full shardedDONs and shardsConnMgrs directly. - donConfig := shardedDONsToLegacy(shardedDONs[0]) - don := shardsConnMgrs[0][0] switch handlerType { case DummyHandlerType: - return handlers.NewDummyHandler(donConfig, don, hf.lggr) + return handlers.NewDummyHandler(dons, hf.lggr) case WebAPICapabilitiesType: - return capabilities.NewHandler(handlerConfig, donConfig, don, hf.httpClient, hf.lggr) + return capabilities.NewHandler(handlerConfig, dons, hf.httpClient, hf.lggr) case HTTPCapabilityType: - return v2.NewGatewayHandler(handlerConfig, shardedDONs, shardsConnMgrs, hf.httpClient, hf.lggr, hf.lf, hf.httpClientFactory, hf.orgResolver) + return v2.NewGatewayHandler(handlerConfig, dons, hf.httpClient, hf.lggr, hf.lf, hf.httpClientFactory, hf.orgResolver) case VaultHandlerType: + // For backward compatibility, convert sharded config to legacy DONConfig + // using the first DON's first shard. TODO(CRE-1640): migrate to full + // shardedDONs/shardsConnMgrs support. + donConfig := shardedDONsToLegacy(dons.DONs[0]) + don := dons.ConnMgrs[0][0] return vault.NewHandler(handlerConfig, donConfig, don, hf.capabilitiesRegistry, hf.workflowRegistrySyncer, hf.lggr, clockwork.NewRealClock(), hf.lf) case ConfidentialRelayHandlerType: + // For backward compatibility, convert sharded config to legacy DONConfig + // using the first DON's first shard. TODO(CRE-1640): migrate to full + // shardedDONs/shardsConnMgrs support. + donConfig := shardedDONsToLegacy(dons.DONs[0]) + don := dons.ConnMgrs[0][0] return confidentialrelay.NewHandler(handlerConfig, donConfig, don, hf.lggr, clockwork.NewRealClock(), hf.lf) default: return nil, fmt.Errorf("unsupported handler type %s", handlerType) diff --git a/core/services/gateway/handlers/capabilities/handler.go b/core/services/gateway/handlers/capabilities/handler.go index d7523f32abb..4e945313542 100644 --- a/core/services/gateway/handlers/capabilities/handler.go +++ b/core/services/gateway/handlers/capabilities/handler.go @@ -17,7 +17,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/ratelimit" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/common" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network" @@ -35,8 +34,7 @@ const ( type handler struct { services.StateMachine config HandlerConfig - don handlers.DON - donConfig *config.DONConfig + nodeAddrToShard map[string]*handlers.ShardEndpoint lggr logger.Logger httpClient network.HTTPClient nodeRateLimiter *ratelimit.RateLimiter @@ -50,13 +48,22 @@ type HandlerConfig struct { var _ handlers.Handler = (*handler)(nil) -func NewHandler(handlerConfig json.RawMessage, donConfig *config.DONConfig, don handlers.DON, httpClient network.HTTPClient, lggr logger.Logger) (*handler, error) { +func NewHandler(handlerConfig json.RawMessage, dons *handlers.ShardedDONs, httpClient network.HTTPClient, lggr logger.Logger) (*handler, error) { var cfg HandlerConfig err := json.Unmarshal(handlerConfig, &cfg) if err != nil { return nil, err } + _, nodeAddrToShard, err := dons.BuildShardEndpoints() + if err != nil { + return nil, err + } + defaultDonID := "" + if len(dons.DONs) > 0 { + defaultDonID = dons.DONs[0].DonName + } + nodeRateLimiter, err := ratelimit.NewRateLimiter(cfg.NodeRateLimiter) if err != nil { return nil, err @@ -69,9 +76,8 @@ func NewHandler(handlerConfig json.RawMessage, donConfig *config.DONConfig, don return &handler{ config: cfg, - don: don, - donConfig: donConfig, - lggr: logger.Named(lggr, "WebAPIHandler."+donConfig.DonID), + nodeAddrToShard: nodeAddrToShard, + lggr: logger.Named(lggr, "WebAPIHandler."+defaultDonID), httpClient: httpClient, nodeRateLimiter: nodeRateLimiter, metrics: metrics, @@ -181,7 +187,12 @@ func (h *handler) handleWebAPIOutgoingMessage(ctx context.Context, msg *api.Mess l.Errorw(ErrTransformingMessageToRequest, "err", err) return } - err = h.don.SendToNode(newCtx, nodeAddr, req) + shard, ok := h.nodeAddrToShard[nodeAddr] + if !ok { + l.Errorw("no connection manager found for node", "to", nodeAddr) + return + } + err = shard.ConnMgr.SendToNode(newCtx, nodeAddr, req) h.metrics.recordArtifactFetchResponseDelivery(newCtx, err == nil) if err != nil { l.Errorw("failed to send to node", "err", err, "to", nodeAddr) diff --git a/core/services/gateway/handlers/capabilities/handler_test.go b/core/services/gateway/handlers/capabilities/handler_test.go index 443fc9de0a4..854ae120796 100644 --- a/core/services/gateway/handlers/capabilities/handler_test.go +++ b/core/services/gateway/handlers/capabilities/handler_test.go @@ -16,6 +16,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" gwcommon "github.com/smartcontractkit/chainlink/v2/core/services/gateway/common" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" + "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" hc "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/common" handlermocks "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/mocks" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network" @@ -38,18 +39,23 @@ func setupHandler(t *testing.T) (*handler, *mocks.HTTPClient, *handlermocks.DON, cfgBytes, err := json.Marshal(handlerConfig) require.NoError(t, err) - donConfig := &config.DONConfig{ - Members: []config.NodeConfig{}, - F: 1, - } nodes := gwcommon.NewTestNodes(t, 2) + members := make([]config.NodeConfig, 0, len(nodes)) for id, n := range nodes { - donConfig.Members = append(donConfig.Members, config.NodeConfig{ + members = append(members, config.NodeConfig{ Name: fmt.Sprintf("node_%d", id), Address: n.Address, }) } - handler, err := NewHandler(json.RawMessage(cfgBytes), donConfig, don, httpClient, lggr) + shardedDONs := []config.ShardedDONConfig{ + { + F: 1, + Shards: []config.Shard{{Nodes: members}}, + }, + } + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{don}}) + require.NoError(t, err) + handler, err := NewHandler(json.RawMessage(cfgBytes), dons, httpClient, lggr) require.NoError(t, err) return handler, httpClient, don, nodes } diff --git a/core/services/gateway/handlers/capabilities/v2/http_handler.go b/core/services/gateway/handlers/capabilities/v2/http_handler.go index ceb6ba60c2b..5902a77a33b 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_handler.go +++ b/core/services/gateway/handlers/capabilities/v2/http_handler.go @@ -18,7 +18,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/capabilities/v2/metrics" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network" @@ -46,8 +45,8 @@ const ( type gatewayHandler struct { services.StateMachine config ServiceConfig - shards []*shardEndpoint // all DON shards served by this handler, across the full DON×shard matrix - nodeAddrToShard map[string]*shardEndpoint // node address -> owning shard, for routing responses back to the correct shard conn manager + shards []*handlers.ShardEndpoint // all DON shards served by this handler, across the full DON×shard matrix + nodeAddrToShard map[string]*handlers.ShardEndpoint // node address -> owning shard, for routing responses back to the correct shard conn manager lggr logger.Logger httpClient network.HTTPClient wg sync.WaitGroup @@ -122,7 +121,7 @@ type RetryConfig struct { Multiplier float64 `json:"multiplier"` } -func NewGatewayHandler(handlerConfig json.RawMessage, shardedDONs []config.ShardedDONConfig, shardsConnMgrs [][]handlers.DON, httpClient network.HTTPClient, lggr logger.Logger, lf limits.Factory, httpClientFactory network.HTTPClientFactory, orgResolver orgresolver.OrgResolver) (*gatewayHandler, error) { +func NewGatewayHandler(handlerConfig json.RawMessage, dons *handlers.ShardedDONs, httpClient network.HTTPClient, lggr logger.Logger, lf limits.Factory, httpClientFactory network.HTTPClientFactory, orgResolver orgresolver.OrgResolver) (*gatewayHandler, error) { var cfg ServiceConfig err := json.Unmarshal(handlerConfig, &cfg) if err != nil { @@ -130,11 +129,11 @@ func NewGatewayHandler(handlerConfig json.RawMessage, shardedDONs []config.Shard } cfg = WithDefaults(cfg) - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, shardsConnMgrs) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() if err != nil { return nil, fmt.Errorf("failed to build shard endpoints: %w", err) } - members := allMembers(shards) + members := handlers.AllMembers(shards) globalNodeRateLimiter, err := lf.MakeRateLimiter(cresettings.Default.GatewayHTTPGlobalRate) if err != nil { @@ -614,11 +613,11 @@ func (h *gatewayHandler) sendResponseToNode(ctx context.Context, requestID strin if !ok { return fmt.Errorf("cannot route response to unknown node %s (no owning shard)", nodeAddr) } - err = shard.connMgr.SendToNode(ctx, nodeAddr, req) + err = shard.ConnMgr.SendToNode(ctx, nodeAddr, req) if err != nil { return err } - h.lggr.Debugw("sent response to node", "to", nodeAddr, "shard", shard.donID) + h.lggr.Debugw("sent response to node", "to", nodeAddr, "shard", shard.DonID) return nil } diff --git a/core/services/gateway/handlers/capabilities/v2/http_handler_test.go b/core/services/gateway/handlers/capabilities/v2/http_handler_test.go index f46fae5d4b6..703775280f2 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_handler_test.go +++ b/core/services/gateway/handlers/capabilities/v2/http_handler_test.go @@ -44,8 +44,8 @@ func TestNewGatewayHandler(t *testing.T) { mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) require.NotNil(t, handler.responseCache) @@ -60,8 +60,8 @@ func TestNewGatewayHandler(t *testing.T) { mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(invalidConfig, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(invalidConfig, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.Error(t, err) require.Nil(t, handler) }) @@ -78,8 +78,8 @@ func TestNewGatewayHandler(t *testing.T) { mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) require.Equal(t, defaultCleanUpPeriodMs, handler.config.CleanUpPeriodMs) // Default value @@ -90,7 +90,7 @@ func TestHandleNodeMessage(t *testing.T) { handler, _ := createTestHandler(t) t.Run("successful node message handling", func(t *testing.T) { - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) // Prepare outbound request @@ -131,7 +131,7 @@ func TestHandleNodeMessage(t *testing.T) { }) t.Run("successful node message handling with MultiHeaders", func(t *testing.T) { - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) // Prepare outbound request @@ -229,7 +229,7 @@ func TestHandleNodeMessage(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) // First call: should fetch from HTTP client and cache the response httpResp := &network.HTTPResponse{ StatusCode: 200, @@ -275,7 +275,7 @@ func TestHandleNodeMessage(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) httpResp := &network.HTTPResponse{ StatusCode: 500, @@ -446,8 +446,8 @@ func TestGatewayHandler_Start_CallsDeleteExpired(t *testing.T) { mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) mockCache := newMockResponseCache() @@ -475,10 +475,13 @@ func serviceCfg() ServiceConfig { // shardedArgs builds the multi-shard arguments for NewGatewayHandler from a // legacy single-shard DONConfig and its connection manager, producing a // one-DON one-shard matrix. -func shardedArgs(donConfig *config.DONConfig, mockDon *handlermocks.DON) ([]config.ShardedDONConfig, [][]handlers.DON) { - return []config.ShardedDONConfig{ +func shardedArgs(t *testing.T, donConfig *config.DONConfig, mockDon *handlermocks.DON) *handlers.ShardedDONs { + shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, - }, [][]handlers.DON{{mockDon}} + } + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + return dons } func createTestHandler(t *testing.T) (*gatewayHandler, *sdkmetric.ManualReader) { @@ -509,11 +512,11 @@ func createTestHandlerWithConfig(t *testing.T, cfg ServiceConfig) (*gatewayHandl mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) meter, reader := newBeholderReader(t) - handler.metrics, err = metrics.NewMetricsWithMeter(allMembers(handler.shards), meter) + handler.metrics, err = metrics.NewMetricsWithMeter(handlers.AllMembers(handler.shards), meter) require.NoError(t, err) require.NotNil(t, handler) @@ -700,7 +703,7 @@ func TestCreateHTTPRequestCallback(t *testing.T) { func TestMakeOutgoingRequest_SendResponseUsesIndependentContext(t *testing.T) { handler, _ := createTestHandler(t) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) outboundReq := gateway_common.OutboundHTTPRequest{ @@ -801,7 +804,7 @@ func TestHTTPActionLatencyMetrics(t *testing.T) { t.Run("async timer covers HTTP work plus response send; send has its own timer", func(t *testing.T) { t.Parallel() handler, reader := createTestHandler(t) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) const httpDelay = 120 * time.Millisecond @@ -843,7 +846,7 @@ func TestHTTPActionLatencyMetrics(t *testing.T) { t.Run("failed outbound call still records endpoint latency", func(t *testing.T) { t.Parallel() handler, reader := createTestHandler(t) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) const httpDelay = 100 * time.Millisecond @@ -869,7 +872,7 @@ func TestHTTPActionLatencyMetrics(t *testing.T) { t.Run("cache hit emits no outbound-call observation", func(t *testing.T) { t.Parallel() handler, reader := createTestHandler(t) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockCache := newMockResponseCache() mockCache.fetchResp = &gateway_common.OutboundHTTPResponse{StatusCode: 200} @@ -930,7 +933,7 @@ func TestMakeOutgoingRequestCachingBehavior(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) httpResp := &network.HTTPResponse{ StatusCode: 200, @@ -974,7 +977,7 @@ func TestMakeOutgoingRequestCachingBehavior(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) httpResp := &network.HTTPResponse{ StatusCode: 200, @@ -1018,7 +1021,7 @@ func TestMakeOutgoingRequestCachingBehavior(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) httpResp := &network.HTTPResponse{ StatusCode: 200, @@ -1060,7 +1063,7 @@ func setupRateLimitingTest(t *testing.T, cfg ServiceConfig) (*gatewayHandler, *j } mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) return handler, resp, mockHTTPClient, mockDon } @@ -1399,10 +1402,12 @@ func TestNewGatewayHandler_MultiShardCreatesRateLimitersForAllMembers(t *testing {handlermocks.NewDON(t), handlermocks.NewDON(t)}, {handlermocks.NewDON(t), handlermocks.NewDON(t)}, } + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) @@ -1410,10 +1415,10 @@ func TestNewGatewayHandler_MultiShardCreatesRateLimitersForAllMembers(t *testing require.Len(t, handler.shards, 4) // Verify donIDs: shard 0 = bare name, shard 1 = suffixed. - require.Equal(t, "donA", handler.shards[0].donID) - require.Equal(t, "donA_1", handler.shards[1].donID) - require.Equal(t, "donB", handler.shards[2].donID) - require.Equal(t, "donB_1", handler.shards[3].donID) + require.Equal(t, "donA", handler.shards[0].DonID) + require.Equal(t, "donA_1", handler.shards[1].DonID) + require.Equal(t, "donB", handler.shards[2].DonID) + require.Equal(t, "donB_1", handler.shards[3].DonID) // Verify per-node rate limiters created for all 8 members. require.Len(t, handler.perNodeRateLimiters, 8) @@ -1422,10 +1427,10 @@ func TestNewGatewayHandler_MultiShardCreatesRateLimitersForAllMembers(t *testing } // Verify nodeAddrToShard maps each node to the correct shard endpoint. - require.Equal(t, "donA", handler.nodeAddrToShard["a1"].donID) - require.Equal(t, "donA_1", handler.nodeAddrToShard["a3"].donID) - require.Equal(t, "donB", handler.nodeAddrToShard["b1"].donID) - require.Equal(t, "donB_1", handler.nodeAddrToShard["b3"].donID) + require.Equal(t, "donA", handler.nodeAddrToShard["a1"].DonID) + require.Equal(t, "donA_1", handler.nodeAddrToShard["a3"].DonID) + require.Equal(t, "donB", handler.nodeAddrToShard["b1"].DonID) + require.Equal(t, "donB_1", handler.nodeAddrToShard["b3"].DonID) } // TestGatewayHandler_SendResponseToNode_MultiShardRouting verifies that when a @@ -1449,10 +1454,12 @@ func TestGatewayHandler_SendResponseToNode_MultiShardRouting(t *testing.T) { shard0Don := handlermocks.NewDON(t) shard1Don := handlermocks.NewDON(t) connMgrs := [][]handlers.DON{{shard0Don, shard1Don}} + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) diff --git a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go index ecb77ca6cac..f02c70db5ae 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go +++ b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go @@ -58,8 +58,8 @@ type savedCallback struct { type httpTriggerHandler struct { services.StateMachine config ServiceConfig - shards []*shardEndpoint - nodeAddrToShard map[string]*shardEndpoint + shards []*handlers.ShardEndpoint + nodeAddrToShard map[string]*handlers.ShardEndpoint lggr logger.Logger callbacksMu sync.Mutex callbacks map[string]savedCallback // requestID -> savedCallback @@ -77,7 +77,7 @@ type HTTPTriggerHandler interface { HandleNodeTriggerResponse(ctx context.Context, resp *jsonrpc.Response[json.RawMessage], nodeAddr string) error } -func NewHTTPTriggerHandler(lggr logger.Logger, cfg ServiceConfig, shards []*shardEndpoint, nodeAddrToShard map[string]*shardEndpoint, workflowMetadataHandler *WorkflowMetadataHandler, userRateLimiter limits.RateLimiter, metrics *metrics.Metrics, orgResolver orgresolver.OrgResolver) *httpTriggerHandler { +func NewHTTPTriggerHandler(lggr logger.Logger, cfg ServiceConfig, shards []*handlers.ShardEndpoint, nodeAddrToShard map[string]*handlers.ShardEndpoint, workflowMetadataHandler *WorkflowMetadataHandler, userRateLimiter limits.RateLimiter, metrics *metrics.Metrics, orgResolver orgresolver.OrgResolver) *httpTriggerHandler { return &httpTriggerHandler{ lggr: logger.Named(lggr, "RequestCallbacks"), callbacks: make(map[string]savedCallback), @@ -436,12 +436,12 @@ func (h *httpTriggerHandler) setupCallback(ctx context.Context, requestID string aggregators := make(map[string]*aggregation.IdenticalNodeResponseAggregator, len(assigned)) for _, shard := range assigned { // (N+F)//2 + 1 threshold where N = number of nodes, F = number of faulty nodes - threshold := (len(shard.members)+shard.f)/2 + 1 + threshold := (len(shard.Members)+shard.F)/2 + 1 agg, err := aggregation.NewIdenticalNodeResponseAggregator(threshold) if err != nil { return nil, errors.New("failed to create response aggregator: " + err.Error()) } - aggregators[shard.donID] = agg + aggregators[shard.DonID] = agg } doneCh := make(chan struct{}) @@ -489,18 +489,18 @@ func (h *httpTriggerHandler) HandleNodeTriggerResponse(ctx context.Context, resp if !ok { return fmt.Errorf("received trigger response from unknown node %s (no owning shard)", nodeAddr) } - agg, ok := saved.responseAggregators[shard.donID] + agg, ok := saved.responseAggregators[shard.DonID] if !ok { // The node belongs to a shard this workflow isn't assigned to (or the // callback was captured before the workflow was assigned there). - return fmt.Errorf("node %s (shard %s) is not assigned to workflow for request ID %s", nodeAddr, shard.donID, resp.ID) + return fmt.Errorf("node %s (shard %s) is not assigned to workflow for request ID %s", nodeAddr, shard.DonID, resp.ID) } aggResp, err := agg.CollectAndAggregate(resp, nodeAddr) if err != nil { return err } if aggResp == nil { - h.lggr.Debugw("Not enough responses to aggregate", "requestID", resp.ID, "nodeAddress", nodeAddr, "shard", shard.donID) + h.lggr.Debugw("Not enough responses to aggregate", "requestID", resp.ID, "nodeAddress", nodeAddr, "shard", shard.DonID) return nil } rawResp, err := json.Marshal(aggResp) @@ -683,7 +683,7 @@ func (h *httpTriggerHandler) sendWithRetries(ctx context.Context, legacyExecutio // sendToShard sends the request to all members of a single shard, retrying // failures until all succeed, the callback is responded to (doneCh), or ctx is // cancelled (overall max duration). -func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoint, legacyExecutionID, executionIDWithTriggerIndex string, req *jsonrpc.Request[json.RawMessage], doneCh <-chan struct{}) error { +func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *handlers.ShardEndpoint, legacyExecutionID, executionIDWithTriggerIndex string, req *jsonrpc.Request[json.RawMessage], doneCh <-chan struct{}) error { nodeTimeout := time.Duration(h.config.NodeSendTimeoutMs) * time.Millisecond successfulNodes := make(map[string]bool) @@ -696,7 +696,7 @@ func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoi for { var pending []string - for _, member := range shard.members { + for _, member := range shard.Members { if !successfulNodes[member.Address] { pending = append(pending, member.Address) } @@ -715,7 +715,7 @@ func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoi h.metrics.IncrementTriggerCapabilityRequestCount(ctx, nodeAddress, gateway_common.MethodWorkflowExecute, h.lggr) sendStart := time.Now() - err := shard.connMgr.SendToNode(nodeCtx, nodeAddress, req) + err := shard.ConnMgr.SendToNode(nodeCtx, nodeAddress, req) h.metrics.RecordGatewayToNodeLatency(ctx, time.Since(sendStart).Milliseconds(), nodeAddress, gateway_common.MethodWorkflowExecute, h.lggr) if err != nil { h.metrics.IncrementTriggerCapabilityRequestFailures(ctx, nodeAddress, gateway_common.MethodWorkflowExecute, h.lggr) @@ -733,7 +733,7 @@ func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoi combinedErr = errors.Join(combinedErr, fmt.Errorf("node %s: %w", res.nodeAddress, res.err)) h.lggr.Debugw("Failed to send trigger request to node, will retry", "node", res.nodeAddress, - "shard", shard.donID, + "shard", shard.DonID, "legacyExecutionID", legacyExecutionID, "executionIDWithTriggerIndex", executionIDWithTriggerIndex, "error", res.err) @@ -742,38 +742,38 @@ func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoi } } - if len(successfulNodes) == len(shard.members) { + if len(successfulNodes) == len(shard.Members) { h.lggr.Infow("Successfully sent trigger request to all nodes in shard", - "shard", shard.donID, + "shard", shard.DonID, "legacyExecutionID", legacyExecutionID, "executionIDWithTriggerIndex", executionIDWithTriggerIndex, - "nodeCount", len(shard.members)) + "nodeCount", len(shard.Members)) return nil } // Not all nodes succeeded, wait and retry h.lggr.Debugw("Retrying failed nodes for trigger request", - "shard", shard.donID, + "shard", shard.DonID, "legacyExecutionID", legacyExecutionID, "executionIDWithTriggerIndex", executionIDWithTriggerIndex, - "failedCount", len(shard.members)-len(successfulNodes), + "failedCount", len(shard.Members)-len(successfulNodes), "errors", combinedErr) select { case <-doneCh: h.lggr.Infow("Callback already responded to, stopping retries", - "shard", shard.donID, + "shard", shard.DonID, "legacyExecutionID", legacyExecutionID, "executionIDWithTriggerIndex", executionIDWithTriggerIndex, "requestID", req.ID, "successNodes", len(successfulNodes), - "totalNodes", len(shard.members)) + "totalNodes", len(shard.Members)) return nil case <-time.After(b.Duration()): continue case <-ctx.Done(): return fmt.Errorf("shard %s: request retry time exceeded, some nodes may not have received the request: legacyExecutionID=%s, executionIDWithTriggerIndex=%s, successNodes=%d, totalNodes=%d", - shard.donID, legacyExecutionID, executionIDWithTriggerIndex, len(successfulNodes), len(shard.members)) + shard.DonID, legacyExecutionID, executionIDWithTriggerIndex, len(successfulNodes), len(shard.Members)) } } } diff --git a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler_test.go b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler_test.go index e31d650d065..81127321a4b 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler_test.go +++ b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler_test.go @@ -545,7 +545,7 @@ func registerWorkflow(_ *testing.T, handler *httpTriggerHandler, workflowID stri func assignWorkflowToAllShards(mh *WorkflowMetadataHandler, workflowID string) { mh.mu.Lock() defer mh.mu.Unlock() - mh.workflowShards[workflowID] = append([]*shardEndpoint(nil), mh.shards...) + mh.workflowShards[workflowID] = append([]*handlers.ShardEndpoint(nil), mh.shards...) } // registerWorkflowOnShards is the sharding-aware variant of registerWorkflow. @@ -553,13 +553,13 @@ func assignWorkflowToAllShards(mh *WorkflowMetadataHandler, workflowID string) { // workflow to the given shard endpoints so that WorkflowShards(workflowID) // returns them and the trigger handler fans the request out only to those // shards. The metadata handler's map is written directly under its mutex. -func registerWorkflowOnShards(t *testing.T, handler *httpTriggerHandler, workflowID string, privateKey *ecdsa.PrivateKey, assignedShards ...*shardEndpoint) { +func registerWorkflowOnShards(t *testing.T, handler *httpTriggerHandler, workflowID string, privateKey *ecdsa.PrivateKey, assignedShards ...*handlers.ShardEndpoint) { t.Helper() registerWorkflow(t, handler, workflowID, privateKey) handler.workflowMetadataHandler.mu.Lock() defer handler.workflowMetadataHandler.mu.Unlock() - assigned := make([]*shardEndpoint, len(assignedShards)) + assigned := make([]*handlers.ShardEndpoint, len(assignedShards)) copy(assigned, assignedShards) handler.workflowMetadataHandler.workflowShards[workflowID] = assigned } @@ -933,15 +933,17 @@ func TestHttpTriggerHandler_HandleUserTriggerRequest_DeliversOnlyToRegisteredSha shard2Don := handlermocks.NewDON(t) connMgrs := [][]handlers.DON{{shard0Don, shard1Don, shard2Don}} - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, connMgrs) + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) require.Len(t, shards, 3) // Sanity-check donID derivation used by the metadata/trigger handlers. - require.Equal(t, "don", shards[0].donID) - require.Equal(t, "don_1", shards[1].donID) - require.Equal(t, "don_2", shards[2].donID) + require.Equal(t, "don", shards[0].DonID) + require.Equal(t, "don_1", shards[1].DonID) + require.Equal(t, "don_2", shards[2].DonID) - allMembersSlice := allMembers(shards) + allMembersSlice := handlers.AllMembers(shards) testMetrics, err := metrics.NewMetrics(allMembersSlice) require.NoError(t, err) metadataHandler := NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics) @@ -1005,9 +1007,9 @@ func TestHttpTriggerHandler_HandleUserTriggerRequest_JWTAuthorization(t *testing ctx := t.Context() // Setup metadata handler with test data - err := handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].donID].Start(ctx) + err := handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].donID].Close() + defer handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].DonID].Close() // Create test keys privateKey := createTestPrivateKey(t) @@ -1153,9 +1155,9 @@ func TestHttpTriggerHandler_HandleUserTriggerRequest_WorkflowLookup(t *testing.T handler, mockDon := createTestTriggerHandler(t) ctx := t.Context() - err := handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].donID].Start(ctx) + err := handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].donID].Close() + defer handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].DonID].Close() privateKey := createTestPrivateKey(t) signerAddr := crypto.PubkeyToAddress(privateKey.PublicKey) @@ -1838,7 +1840,9 @@ func createTestMetadataHandler(t *testing.T) *WorkflowMetadataHandler { shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) return NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics) } @@ -1851,7 +1855,9 @@ func newTestTriggerHandler(t *testing.T, lggr logger.Logger, cfg ServiceConfig, shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) // Repoint the metadata handler at THIS handler's shard set so that the // workflowShards populated by registerWorkflow route sends to mockDon. The @@ -1863,8 +1869,8 @@ func newTestTriggerHandler(t *testing.T, lggr logger.Logger, cfg ServiceConfig, metadataHandler.nodeAddrToShard = nodeAddrToShard metadataHandler.aggs = make(map[string]*aggregation.WorkflowMetadataAggregator, len(shards)) for _, shard := range shards { - threshold := shard.f + 1 - metadataHandler.aggs[shard.donID] = aggregation.NewWorkflowMetadataAggregator(metadataHandler.lggr, threshold, time.Duration(cfg.CleanUpPeriodMs)*time.Millisecond, testMetrics) + threshold := shard.F + 1 + metadataHandler.aggs[shard.DonID] = aggregation.NewWorkflowMetadataAggregator(metadataHandler.lggr, threshold, time.Duration(cfg.CleanUpPeriodMs)*time.Millisecond, testMetrics) } return NewHTTPTriggerHandler(lggr, cfg, shards, nodeAddrToShard, metadataHandler, userRateLimiter, testMetrics, nil) } @@ -1892,14 +1898,16 @@ func createTestTriggerHandlerWithConfig(t *testing.T, cfg ServiceConfig) (*httpT testMetrics := createTestMetrics(t, donConfig) // Build ONE shared shard set so the metadata handler and the trigger handler - // reference the SAME shardEndpoint instances (and the same mockDon). This is + // reference the SAME ShardEndpoint instances (and the same mockDon). This is // required because registerWorkflow writes workflowShards to the metadata // handler's .shards, and sendWithRetries reads them back and sends via - // shard.connMgr — both must hit mockDon, which the test sets expectations on. + // shard.ConnMgr — both must hit mockDon, which the test sets expectations on. shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) metadataHandler := NewWorkflowMetadataHandler(lggr, WithDefaults(cfg), shards, nodeAddrToShard, testMetrics) @@ -2223,7 +2231,7 @@ func TestHttpTriggerHandler_HandleUserTriggerRequest_StopsRetriesOnQuorum(t *tes // createMultiShardTriggerHandler builds a trigger handler over multiple shards // of a single DON. Each shard gets its own mock DON connection manager. Returns // the handler, the mock DONs (one per shard), and the shard endpoints. -func createMultiShardTriggerHandler(t *testing.T, donName string, shardNodeSets [][]string, f int) (*httpTriggerHandler, []*handlermocks.DON, []*shardEndpoint) { +func createMultiShardTriggerHandler(t *testing.T, donName string, shardNodeSets [][]string, f int) (*httpTriggerHandler, []*handlermocks.DON, []*handlers.ShardEndpoint) { t.Helper() lggr := logger.Test(t) cfg := WithDefaults(ServiceConfig{ @@ -2247,10 +2255,12 @@ func createMultiShardTriggerHandler(t *testing.T, donName string, shardNodeSets shardedDONs := []config.ShardedDONConfig{ {DonName: donName, F: f, Shards: shardsCfg}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{connMgrs}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{connMgrs}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) - allMembersSlice := allMembers(shards) + allMembersSlice := handlers.AllMembers(shards) testMetrics, err := metrics.NewMetrics(allMembersSlice) require.NoError(t, err) metadataHandler := NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics) @@ -2460,10 +2470,12 @@ func TestHttpTriggerHandler_MultiShardSendFailureResilience(t *testing.T) { shard0Don := handlermocks.NewDON(t) shard1Don := handlermocks.NewDON(t) connMgrs := [][]handlers.DON{{shard0Don, shard1Don}} - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, connMgrs) + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) - allMembersSlice := allMembers(shards) + allMembersSlice := handlers.AllMembers(shards) testMetrics, err := metrics.NewMetrics(allMembersSlice) require.NoError(t, err) metadataHandler := NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics) diff --git a/core/services/gateway/handlers/capabilities/v2/outbound_bounds_test.go b/core/services/gateway/handlers/capabilities/v2/outbound_bounds_test.go index 5c76b42e902..5ed77d3c82f 100644 --- a/core/services/gateway/handlers/capabilities/v2/outbound_bounds_test.go +++ b/core/services/gateway/handlers/capabilities/v2/outbound_bounds_test.go @@ -83,7 +83,7 @@ func TestMakeOutgoingRequest_NodeLimitIsolatesPeers(t *testing.T) { t.Parallel() handler, _ := createTestHandler(t) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient.EXPECT().Send(mock.Anything, mock.Anything).Return(okHTTPResponse(), nil).Once() mockDon.EXPECT().SendToNode(mock.Anything, "node2", mock.Anything).Return(nil) @@ -106,7 +106,7 @@ func TestMakeOutgoingRequest_ReleasesSlotOnCompletion(t *testing.T) { t.Parallel() handler, _ := createTestHandler(t) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient.EXPECT().Send(mock.Anything, mock.Anything).Return(okHTTPResponse(), nil).Times(3) mockDon.EXPECT().SendToNode(mock.Anything, "node1", mock.Anything).Return(nil) @@ -133,7 +133,7 @@ func TestMakeOutgoingRequest_ReleasesSlotOnFailure(t *testing.T) { t.Parallel() handler, _ := createTestHandler(t) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient.EXPECT().Send(mock.Anything, mock.Anything).Return(nil, errors.New("upstream down")).Once() mockDon.EXPECT().SendToNode(mock.Anything, "node1", mock.Anything).Return(nil) diff --git a/core/services/gateway/handlers/capabilities/v2/shard_endpoints.go b/core/services/gateway/handlers/capabilities/v2/shard_endpoints.go deleted file mode 100644 index 5404b86d9a8..00000000000 --- a/core/services/gateway/handlers/capabilities/v2/shard_endpoints.go +++ /dev/null @@ -1,77 +0,0 @@ -package v2 - -import ( - "errors" - "fmt" - - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" -) - -type shardEndpoint struct { - // donID is the shard-specific DON identifier, derived via config.ShardDONID. - // Shard 0 uses the bare DON name; shard N>0 uses "donName_N". - donID string - donName string - shardIdx int - // connMgr is the connection manager for this shard. It only knows how to reach - // nodes that are members of this shard. - connMgr handlers.DON - members []config.NodeConfig - f int -} - -// buildShardEndpoints expands the full DON×shard matrix into a flat list of -// shardEndpoints, and builds a nodeAddr -> shardEndpoint lookup. The outer -// dimension of shardsConnMgrs indexes DONs (parallel to shardedDONs); the inner -// dimension indexes shards within that DON (parallel to shardedDONs[i].Shards). -// Memberships must be disjoint. -func buildShardEndpoints(shardedDONs []config.ShardedDONConfig, shardsConnMgrs [][]handlers.DON) ([]*shardEndpoint, map[string]*shardEndpoint, error) { - if len(shardedDONs) == 0 || len(shardsConnMgrs) == 0 { - return nil, nil, errors.New("at least one DON and connection manager required") - } - if len(shardedDONs) != len(shardsConnMgrs) { - return nil, nil, fmt.Errorf("shardedDONs length %d does not match shardsConnMgrs length %d", len(shardedDONs), len(shardsConnMgrs)) - } - - var endpoints []*shardEndpoint - addrToShard := make(map[string]*shardEndpoint) - - for i, sd := range shardedDONs { - connMgrs := shardsConnMgrs[i] - if len(sd.Shards) != len(connMgrs) { - return nil, nil, fmt.Errorf("DON %s: shards length %d does not match connection managers length %d", sd.DonName, len(sd.Shards), len(connMgrs)) - } - for shardIdx, shard := range sd.Shards { - if connMgrs[shardIdx] == nil { - return nil, nil, fmt.Errorf("DON %s shard %d: nil connection manager", sd.DonName, shardIdx) - } - ep := &shardEndpoint{ - donID: config.ShardDONID(sd.DonName, shardIdx), - donName: sd.DonName, - shardIdx: shardIdx, - connMgr: connMgrs[shardIdx], - members: shard.Nodes, - f: sd.F, - } - endpoints = append(endpoints, ep) - for _, member := range shard.Nodes { - if existing, ok := addrToShard[member.Address]; ok { - return nil, nil, fmt.Errorf("node address %s appears in both %s and %s; shard memberships must be disjoint", member.Address, existing.donID, ep.donID) - } - addrToShard[member.Address] = ep - } - } - } - - return endpoints, addrToShard, nil -} - -// allMembers returns the union of all shard members across the matrix. -func allMembers(endpoints []*shardEndpoint) []config.NodeConfig { - var out []config.NodeConfig - for _, ep := range endpoints { - out = append(out, ep.members...) - } - return out -} diff --git a/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler.go b/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler.go index 43b2e39b4a9..609834361e7 100644 --- a/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler.go +++ b/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler.go @@ -14,6 +14,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/common/aggregation" + "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/capabilities/v2/metrics" "github.com/smartcontractkit/chainlink/v2/core/utils" ) @@ -40,11 +41,11 @@ type WorkflowMetadataHandler struct { authorizedKeys map[string]map[gateway.AuthorizedKey]struct{} // map of workflow ID to authorized keys workflowRefToID map[workflowReference]string // map of workflow reference to workflow ID workflowIDToRef map[string]workflowReference // map of workflow ID to workflow reference - workflowShards map[string][]*shardEndpoint // map of workflow ID to the shards it is assigned to (quorum reached) + workflowShards map[string][]*handlers.ShardEndpoint // map of workflow ID to the shards it is assigned to (quorum reached) // aggs holds one WorkflowMetadataAggregator per shard, keyed by shard donID. aggs map[string]*aggregation.WorkflowMetadataAggregator - shards []*shardEndpoint - nodeAddrToShard map[string]*shardEndpoint + shards []*handlers.ShardEndpoint + nodeAddrToShard map[string]*handlers.ShardEndpoint config ServiceConfig stopCh services.StopChan metrics *metrics.Metrics @@ -55,18 +56,18 @@ type WorkflowMetadataHandler struct { // NewWorkflowMetadataHandler creates a new WorkflowMetadataHandler spanning the // full DON×shard matrix. Each shard gets its own aggregator with threshold F+1. -func NewWorkflowMetadataHandler(lggr logger.Logger, cfg ServiceConfig, shards []*shardEndpoint, nodeAddrToShard map[string]*shardEndpoint, metrics *metrics.Metrics) *WorkflowMetadataHandler { +func NewWorkflowMetadataHandler(lggr logger.Logger, cfg ServiceConfig, shards []*handlers.ShardEndpoint, nodeAddrToShard map[string]*handlers.ShardEndpoint, metrics *metrics.Metrics) *WorkflowMetadataHandler { aggs := make(map[string]*aggregation.WorkflowMetadataAggregator, len(shards)) for _, shard := range shards { - threshold := shard.f + 1 - aggs[shard.donID] = aggregation.NewWorkflowMetadataAggregator(lggr, threshold, time.Duration(cfg.CleanUpPeriodMs)*time.Millisecond, metrics) + threshold := shard.F + 1 + aggs[shard.DonID] = aggregation.NewWorkflowMetadataAggregator(lggr, threshold, time.Duration(cfg.CleanUpPeriodMs)*time.Millisecond, metrics) } return &WorkflowMetadataHandler{ lggr: logger.Named(lggr, "HTTPTriggerWorkflowMetadataHandler"), authorizedKeys: make(map[string]map[gateway.AuthorizedKey]struct{}), workflowRefToID: make(map[workflowReference]string), workflowIDToRef: make(map[string]workflowReference), - workflowShards: make(map[string][]*shardEndpoint), + workflowShards: make(map[string][]*handlers.ShardEndpoint), aggs: aggs, shards: shards, nodeAddrToShard: nodeAddrToShard, @@ -115,10 +116,10 @@ func (h *WorkflowMetadataHandler) syncMetadata(ctx context.Context) { authorizedKeys := make(map[string]map[gateway.AuthorizedKey]struct{}) workflowRefToID := make(map[workflowReference]string) workflowIDToRef := make(map[string]workflowReference) - workflowShards := make(map[string][]*shardEndpoint) + workflowShards := make(map[string][]*handlers.ShardEndpoint) for _, shard := range h.shards { - agg := h.aggs[shard.donID] + agg := h.aggs[shard.DonID] metadata := agg.Aggregate() for _, data := range metadata { workflowID := data.WorkflowSelector.WorkflowID @@ -197,12 +198,12 @@ func (h *WorkflowMetadataHandler) sendMetadataPullRequest() error { } var combinedErr error for _, shard := range h.shards { - for _, member := range shard.members { + for _, member := range shard.Members { h.metrics.IncrementTriggerCapabilityRequestCount(ctx, member.Address, gateway.MethodPullWorkflowMetadata, h.lggr) - err := shard.connMgr.SendToNode(ctx, member.Address, req) + err := shard.ConnMgr.SendToNode(ctx, member.Address, req) if err != nil { h.metrics.IncrementTriggerCapabilityRequestFailures(ctx, member.Address, gateway.MethodPullWorkflowMetadata, h.lggr) - combinedErr = errors.Join(combinedErr, fmt.Errorf("failed to send pull request to node %s (shard %s): %w", member.Address, shard.donID, err)) + combinedErr = errors.Join(combinedErr, fmt.Errorf("failed to send pull request to node %s (shard %s): %w", member.Address, shard.DonID, err)) } } } @@ -263,16 +264,16 @@ func (h *WorkflowMetadataHandler) aggForNode(nodeAddr string) (*aggregation.Work if !ok { return nil, fmt.Errorf("received metadata from unknown node %s (no owning shard)", nodeAddr) } - return h.aggs[shard.donID], nil + return h.aggs[shard.DonID], nil } // WorkflowShards returns the shards a workflow is currently assigned to (those // whose quorum reported the workflow's metadata). The returned slice is a copy. -func (h *WorkflowMetadataHandler) WorkflowShards(workflowID string) []*shardEndpoint { +func (h *WorkflowMetadataHandler) WorkflowShards(workflowID string) []*handlers.ShardEndpoint { h.mu.RLock() defer h.mu.RUnlock() shards := h.workflowShards[workflowID] - out := make([]*shardEndpoint, len(shards)) + out := make([]*handlers.ShardEndpoint, len(shards)) copy(out, shards) return out } @@ -283,8 +284,8 @@ func (h *WorkflowMetadataHandler) Start(ctx context.Context) error { h.lggr.Info("Starting HTTP Trigger Metadata Handler") h.startTime = time.Now() for _, shard := range h.shards { - if err := h.aggs[shard.donID].Start(ctx); err != nil { - return fmt.Errorf("failed to start aggregator for shard %s: %w", shard.donID, err) + if err := h.aggs[shard.DonID].Start(ctx); err != nil { + return fmt.Errorf("failed to start aggregator for shard %s: %w", shard.DonID, err) } } h.runTicker(time.Duration(h.config.MetadataPullIntervalMs)*time.Millisecond, func(ctx context.Context) { @@ -379,8 +380,8 @@ func (h *WorkflowMetadataHandler) Close() error { return h.StopOnce("WorkflowMetadataHandler", func() error { h.lggr.Info("Stopping HTTP Trigger Metadata Handler") for _, shard := range h.shards { - if err := h.aggs[shard.donID].Close(); err != nil { - h.lggr.Errorw("Failed to close WorkflowMetadataAggregator", "shard", shard.donID, "error", err) + if err := h.aggs[shard.DonID].Close(); err != nil { + h.lggr.Errorw("Failed to close WorkflowMetadataAggregator", "shard", shard.DonID, "error", err) } } close(h.stopCh) diff --git a/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler_test.go b/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler_test.go index 42444fb96a1..ac861778a3f 100644 --- a/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler_test.go +++ b/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler_test.go @@ -59,11 +59,13 @@ func createTestWorkflowMetadataHandler(t *testing.T) (*WorkflowMetadataHandler, // singleShardEndpoints builds a one-DON one-shard endpoint matrix from a legacy // DONConfig for tests. -func singleShardEndpoints(t *testing.T, donConfig *config.DONConfig, mockDon *mocks.DON) ([]*shardEndpoint, map[string]*shardEndpoint) { +func singleShardEndpoints(t *testing.T, donConfig *config.DONConfig, mockDon *mocks.DON) ([]*handlers.ShardEndpoint, map[string]*handlers.ShardEndpoint) { shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) return shards, nodeAddrToShard } @@ -77,9 +79,9 @@ func TestSyncMetadata(t *testing.T) { // Start the aggregator to enable data collection ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() // Add some test data to aggregator key := gateway_common.AuthorizedKey{ @@ -97,9 +99,9 @@ func TestSyncMetadata(t *testing.T) { } // Collect enough observations to meet threshold (F+1 = 2) - err = handler.aggs[handler.shards[0].donID].Collect(&observation, "node1") + err = handler.aggs[handler.shards[0].DonID].Collect(&observation, "node1") require.NoError(t, err) - err = handler.aggs[handler.shards[0].donID].Collect(&observation, "node2") + err = handler.aggs[handler.shards[0].DonID].Collect(&observation, "node2") require.NoError(t, err) handler.syncMetadata(t.Context()) @@ -125,9 +127,9 @@ func TestSyncMetadataMultipleWorkflows(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() // Add observations for multiple workflows workflows := []string{"workflow1", "workflow2"} @@ -149,9 +151,9 @@ func TestSyncMetadataMultipleWorkflows(t *testing.T) { }, }, } - err = handler.aggs[handler.shards[0].donID].Collect(&observation, "node1") + err = handler.aggs[handler.shards[0].DonID].Collect(&observation, "node1") require.NoError(t, err) - err = handler.aggs[handler.shards[0].donID].Collect(&observation, "node2") + err = handler.aggs[handler.shards[0].DonID].Collect(&observation, "node2") require.NoError(t, err) } } @@ -248,9 +250,11 @@ func createMultiShardMetadataHandler(t *testing.T) (*WorkflowMetadataHandler, *m {Nodes: []config.NodeConfig{{Address: "node3"}, {Address: "node4"}}}, }}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon0, mockDon1}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon0, mockDon1}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) - testMetrics, err := metrics.NewMetrics(allMembers(shards)) + testMetrics, err := metrics.NewMetrics(handlers.AllMembers(shards)) require.NoError(t, err) cfg := WithDefaults(ServiceConfig{}) return NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics), mockDon0, mockDon1 @@ -262,8 +266,8 @@ func TestMultiShardAssignment(t *testing.T) { handler, _, _ := createMultiShardMetadataHandler(t) ctx := t.Context() for _, shard := range handler.shards { - require.NoError(t, handler.aggs[shard.donID].Start(ctx)) - t.Cleanup(func() { _ = handler.aggs[shard.donID].Close() }) + require.NoError(t, handler.aggs[shard.DonID].Start(ctx)) + t.Cleanup(func() { _ = handler.aggs[shard.DonID].Close() }) } obs := gateway_common.WorkflowMetadata{ @@ -280,14 +284,14 @@ func TestMultiShardAssignment(t *testing.T) { require.Empty(t, handler.WorkflowShards(testWorkflowID1)) // Quorum (F+1=1) reached only in shard 1 (node3). - require.NoError(t, handler.aggs[handler.shards[1].donID].Collect(&obs, "node3")) + require.NoError(t, handler.aggs[handler.shards[1].DonID].Collect(&obs, "node3")) handler.syncMetadata(ctx) assigned := handler.WorkflowShards(testWorkflowID1) require.Len(t, assigned, 1) - require.Equal(t, handler.shards[1].donID, assigned[0].donID) + require.Equal(t, handler.shards[1].DonID, assigned[0].DonID) // Later quorum reached in shard 0 too (node1) => assigned to both shards. - require.NoError(t, handler.aggs[handler.shards[0].donID].Collect(&obs, "node1")) + require.NoError(t, handler.aggs[handler.shards[0].DonID].Collect(&obs, "node1")) handler.syncMetadata(ctx) assigned = handler.WorkflowShards(testWorkflowID1) require.Len(t, assigned, 2) @@ -316,9 +320,9 @@ func TestOnMetadataPush(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() metadata := gateway_common.WorkflowMetadata{ WorkflowSelector: gateway_common.WorkflowSelector{ @@ -374,9 +378,9 @@ func TestOnMetadataPullResponse(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() key1 := gateway_common.AuthorizedKey{ KeyType: gateway_common.KeyTypeECDSAEVM, @@ -903,9 +907,9 @@ func TestOnMetadataPushWithValidation(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() t.Run("valid metadata passes validation", func(t *testing.T) { metadata := gateway_common.WorkflowMetadata{ @@ -969,9 +973,9 @@ func TestOnMetadataPullResponseWithValidation(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() t.Run("valid metadata array passes validation", func(t *testing.T) { metadata := []gateway_common.WorkflowMetadata{ diff --git a/core/services/gateway/handlers/handler.dummy.go b/core/services/gateway/handlers/handler.dummy.go index ecc991627c6..f525db2a887 100644 --- a/core/services/gateway/handlers/handler.dummy.go +++ b/core/services/gateway/handlers/handler.dummy.go @@ -10,13 +10,12 @@ import ( jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" ) // DummyHandler forwards each request/response without doing any checks. type dummyHandler struct { - donConfig *config.DONConfig - don DON + shards []*ShardEndpoint + defaultDonID string savedCallbacks map[string]*savedCallback mu sync.Mutex lggr logger.Logger @@ -29,12 +28,23 @@ type savedCallback struct { var _ Handler = (*dummyHandler)(nil) -func NewDummyHandler(donConfig *config.DONConfig, don DON, lggr logger.Logger) (Handler, error) { +// NewDummyHandler builds a handler that forwards each request to every node across +// every DON and shard it is given, and relays the first node response back to the +// caller. +func NewDummyHandler(dons *ShardedDONs, lggr logger.Logger) (Handler, error) { + shards, _, err := dons.BuildShardEndpoints() + if err != nil { + return nil, err + } + defaultDonID := "" + if len(dons.DONs) > 0 { + defaultDonID = dons.DONs[0].DonName + } return &dummyHandler{ - donConfig: donConfig, - don: don, + shards: shards, + defaultDonID: defaultDonID, savedCallbacks: make(map[string]*savedCallback), - lggr: logger.Named(lggr, "DummyHandler."+donConfig.DonID), + lggr: logger.Named(lggr, "DummyHandler."+defaultDonID), }, nil } @@ -54,7 +64,7 @@ func (d *dummyHandler) HandleJSONRPCUserMessage(ctx context.Context, jsonRequest msg.Body.Method = jsonRequest.Method } if msg.Body.DonID == "" { - msg.Body.DonID = d.donConfig.DonID + msg.Body.DonID = d.defaultDonID } return d.HandleLegacyUserMessage(ctx, &msg, callback) } @@ -62,7 +72,6 @@ func (d *dummyHandler) HandleJSONRPCUserMessage(ctx context.Context, jsonRequest func (d *dummyHandler) HandleLegacyUserMessage(ctx context.Context, msg *api.Message, callback Callback) error { d.mu.Lock() d.savedCallbacks[msg.Body.MessageID] = &savedCallback{msg.Body.MessageID, callback} - don := d.don d.mu.Unlock() params, err := json.Marshal(msg) if err != nil { @@ -75,8 +84,10 @@ func (d *dummyHandler) HandleLegacyUserMessage(ctx context.Context, msg *api.Mes Method: msg.Body.Method, Params: &rawParams, } - for _, member := range d.donConfig.Members { - err = errors.Join(err, don.SendToNode(ctx, member.Address, req)) + for _, shard := range d.shards { + for _, member := range shard.Members { + err = errors.Join(err, shard.ConnMgr.SendToNode(ctx, member.Address, req)) + } } return err } diff --git a/core/services/gateway/handlers/handler.dummy_test.go b/core/services/gateway/handlers/handler.dummy_test.go index e69bdfced28..6c1e470d9c8 100644 --- a/core/services/gateway/handlers/handler.dummy_test.go +++ b/core/services/gateway/handlers/handler.dummy_test.go @@ -37,15 +37,24 @@ func (m *testConnManager) SendToNode(ctx context.Context, nodeAddress string, re func TestDummyHandler_BasicFlow(t *testing.T) { t.Parallel() - config := config.DONConfig{ - Members: []config.NodeConfig{ - {Name: "node one", Address: "addr_1"}, - {Name: "node two", Address: "addr_2"}, + shardedDONs := []config.ShardedDONConfig{ + { + DonName: "test_don", + Shards: []config.Shard{ + { + Nodes: []config.NodeConfig{ + {Name: "node one", Address: "addr_1"}, + {Name: "node two", Address: "addr_2"}, + }, + }, + }, }, } connMgr := testConnManager{} - handler, err := handlers.NewDummyHandler(&config, &connMgr, logger.Test(t)) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{&connMgr}}) + require.NoError(t, err) + handler, err := handlers.NewDummyHandler(dons, logger.Test(t)) require.NoError(t, err) connMgr.SetHandler("", handler) diff --git a/core/services/gateway/handlers/shard_endpoints.go b/core/services/gateway/handlers/shard_endpoints.go new file mode 100644 index 00000000000..09cd242fa04 --- /dev/null +++ b/core/services/gateway/handlers/shard_endpoints.go @@ -0,0 +1,99 @@ +package handlers + +import ( + "errors" + "fmt" + + "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" +) + +// ShardedDONs pairs a DON×shard config matrix with the connection managers that +// serve it. The outer dimension of ConnMgrs indexes DONs (parallel to DONs); the +// inner dimension indexes shards within that DON (parallel to DONs[i].Shards). +// Use NewShardedDONs to build one: it checks the two stay in lockstep once, so +// callers can pass a single value around instead of two slices that must always +// agree in length. +type ShardedDONs struct { + DONs []config.ShardedDONConfig + ConnMgrs [][]DON +} + +// NewShardedDONs validates that shardedDONs and shardsConnMgrs describe the same +// DON×shard matrix (equal outer length, equal per-DON shard counts, no nil +// connection managers) and bundles them together. +func NewShardedDONs(shardedDONs []config.ShardedDONConfig, shardsConnMgrs [][]DON) (*ShardedDONs, error) { + if len(shardedDONs) == 0 || len(shardsConnMgrs) == 0 { + return nil, errors.New("at least one DON and connection manager required") + } + if len(shardedDONs) != len(shardsConnMgrs) { + return nil, fmt.Errorf("shardedDONs length %d does not match shardsConnMgrs length %d", len(shardedDONs), len(shardsConnMgrs)) + } + for i, sd := range shardedDONs { + connMgrs := shardsConnMgrs[i] + if len(sd.Shards) != len(connMgrs) { + return nil, fmt.Errorf("DON %s: shards length %d does not match connection managers length %d", sd.DonName, len(sd.Shards), len(connMgrs)) + } + for shardIdx, connMgr := range connMgrs { + if connMgr == nil { + return nil, fmt.Errorf("DON %s shard %d: nil connection manager", sd.DonName, shardIdx) + } + } + } + return &ShardedDONs{DONs: shardedDONs, ConnMgrs: shardsConnMgrs}, nil +} + +// ShardEndpoint is one shard of one DON: its identifier, the connection manager +// that reaches its members, and the config (membership, Byzantine fault +// tolerance threshold F) that applies to it. +type ShardEndpoint struct { + // DonID is the shard-specific DON identifier, derived via config.ShardDONID. + // Shard 0 uses the bare DON name; shard N>0 uses "donName_N". + DonID string + DonName string + ShardIdx int + // ConnMgr is the connection manager for this shard. It only knows how to reach + // nodes that are members of this shard. + ConnMgr DON + Members []config.NodeConfig + F int +} + +// BuildShardEndpoints expands the DON×shard matrix into a flat list of +// ShardEndpoints, and builds a nodeAddr -> ShardEndpoint lookup. Node addresses +// must be unique (disjoint) across the whole matrix. +func (s *ShardedDONs) BuildShardEndpoints() ([]*ShardEndpoint, map[string]*ShardEndpoint, error) { + var endpoints []*ShardEndpoint + addrToShard := make(map[string]*ShardEndpoint) + + for i, sd := range s.DONs { + connMgrs := s.ConnMgrs[i] + for shardIdx, shard := range sd.Shards { + ep := &ShardEndpoint{ + DonID: config.ShardDONID(sd.DonName, shardIdx), + DonName: sd.DonName, + ShardIdx: shardIdx, + ConnMgr: connMgrs[shardIdx], + Members: shard.Nodes, + F: sd.F, + } + endpoints = append(endpoints, ep) + for _, member := range shard.Nodes { + if existing, ok := addrToShard[member.Address]; ok { + return nil, nil, fmt.Errorf("node address %s appears in both %s and %s; shard memberships must be disjoint", member.Address, existing.DonID, ep.DonID) + } + addrToShard[member.Address] = ep + } + } + } + + return endpoints, addrToShard, nil +} + +// AllMembers returns the union of all shard members across the matrix. +func AllMembers(endpoints []*ShardEndpoint) []config.NodeConfig { + var out []config.NodeConfig + for _, ep := range endpoints { + out = append(out, ep.Members...) + } + return out +} diff --git a/core/services/gateway/handlers/capabilities/v2/shard_endpoints_test.go b/core/services/gateway/handlers/shard_endpoints_test.go similarity index 58% rename from core/services/gateway/handlers/capabilities/v2/shard_endpoints_test.go rename to core/services/gateway/handlers/shard_endpoints_test.go index 01dd876468c..d782af55578 100644 --- a/core/services/gateway/handlers/capabilities/v2/shard_endpoints_test.go +++ b/core/services/gateway/handlers/shard_endpoints_test.go @@ -1,4 +1,4 @@ -package v2 +package handlers_test import ( "testing" @@ -10,15 +10,58 @@ import ( handlermocks "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/mocks" ) -// handlersDON is a local alias to keep the test matrix literals readable. -type handlersDON = handlers.DON +func node(addr string) config.NodeConfig { + return config.NodeConfig{Name: addr, Address: addr} +} -func TestBuildShardEndpoints(t *testing.T) { +func TestNewShardedDONs(t *testing.T) { t.Parallel() - node := func(addr string) config.NodeConfig { - return config.NodeConfig{Name: addr, Address: addr} - } + t.Run("rejects empty matrix", func(t *testing.T) { + t.Parallel() + + _, err := handlers.NewShardedDONs(nil, nil) + require.Error(t, err) + }) + + t.Run("rejects ragged outer dimension", func(t *testing.T) { + t.Parallel() + + shardedDONs := []config.ShardedDONConfig{ + {DonName: "donA", Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1")}}}}, + } + _, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{}) + require.Error(t, err) + }) + + t.Run("rejects ragged inner dimension", func(t *testing.T) { + t.Parallel() + + shardedDONs := []config.ShardedDONConfig{ + {DonName: "donA", Shards: []config.Shard{ + {Nodes: []config.NodeConfig{node("n1")}}, + {Nodes: []config.NodeConfig{node("n2")}}, + }}, + } + connMgrs := [][]handlers.DON{{handlermocks.NewDON(t)}} // only 1 conn mgr for 2 shards + _, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.Error(t, err) + }) + + t.Run("rejects nil connection manager", func(t *testing.T) { + t.Parallel() + + shardedDONs := []config.ShardedDONConfig{ + {DonName: "donA", Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1")}}}}, + } + connMgrs := [][]handlers.DON{{nil}} + _, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.Error(t, err) + }) +} + +func TestShardedDONs_BuildShardEndpoints(t *testing.T) { + t.Parallel() t.Run("single DON single shard (backward compatible)", func(t *testing.T) { t.Parallel() @@ -27,14 +70,15 @@ func TestBuildShardEndpoints(t *testing.T) { shardedDONs := []config.ShardedDONConfig{ {DonName: "donA", F: 1, Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1"), node("n2")}}}}, } - connMgrs := [][]handlersDON{{don}} + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{don}}) + require.NoError(t, err) - eps, addrMap, err := buildShardEndpoints(shardedDONs, connMgrs) + eps, addrMap, err := dons.BuildShardEndpoints() require.NoError(t, err) require.Len(t, eps, 1) - require.Equal(t, "donA", eps[0].donID) // shard 0 => bare name - require.Equal(t, 0, eps[0].shardIdx) - require.Equal(t, 1, eps[0].f) + require.Equal(t, "donA", eps[0].DonID) // shard 0 => bare name + require.Equal(t, 0, eps[0].ShardIdx) + require.Equal(t, 1, eps[0].F) require.Len(t, addrMap, 2) require.Same(t, eps[0], addrMap["n1"]) }) @@ -42,7 +86,7 @@ func TestBuildShardEndpoints(t *testing.T) { t.Run("full matrix: two DONs each with two shards", func(t *testing.T) { t.Parallel() - dons := [][]handlersDON{ + connMgrs := [][]handlers.DON{ {handlermocks.NewDON(t), handlermocks.NewDON(t)}, {handlermocks.NewDON(t), handlermocks.NewDON(t)}, } @@ -56,86 +100,56 @@ func TestBuildShardEndpoints(t *testing.T) { {Nodes: []config.NodeConfig{node("b1")}}, }}, } + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) - eps, addrMap, err := buildShardEndpoints(shardedDONs, dons) + eps, addrMap, err := dons.BuildShardEndpoints() require.NoError(t, err) require.Len(t, eps, 4) // donIDs: shard 0 bare, shard 1 suffixed - require.Equal(t, "donA", eps[0].donID) - require.Equal(t, "donA_1", eps[1].donID) - require.Equal(t, "donB", eps[2].donID) - require.Equal(t, "donB_1", eps[3].donID) + require.Equal(t, "donA", eps[0].DonID) + require.Equal(t, "donA_1", eps[1].DonID) + require.Equal(t, "donB", eps[2].DonID) + require.Equal(t, "donB_1", eps[3].DonID) require.Len(t, addrMap, 6) require.Same(t, eps[1], addrMap["a2"]) require.Same(t, eps[3], addrMap["b1"]) - require.Equal(t, 2, addrMap["b1"].f) + require.Equal(t, 2, addrMap["b1"].F) }) - t.Run("rejects empty matrix", func(t *testing.T) { - t.Parallel() - - _, _, err := buildShardEndpoints(nil, nil) - require.Error(t, err) - }) - - t.Run("rejects ragged outer dimension", func(t *testing.T) { - t.Parallel() - - shardedDONs := []config.ShardedDONConfig{ - {DonName: "donA", Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1")}}}}, - } - _, _, err := buildShardEndpoints(shardedDONs, [][]handlersDON{}) - require.Error(t, err) - }) - - t.Run("rejects ragged inner dimension", func(t *testing.T) { + t.Run("rejects duplicate node address across shards", func(t *testing.T) { t.Parallel() shardedDONs := []config.ShardedDONConfig{ {DonName: "donA", Shards: []config.Shard{ {Nodes: []config.NodeConfig{node("n1")}}, - {Nodes: []config.NodeConfig{node("n2")}}, + {Nodes: []config.NodeConfig{node("n1")}}, // duplicate }}, } - connMgrs := [][]handlersDON{{handlermocks.NewDON(t)}} // only 1 conn mgr for 2 shards - _, _, err := buildShardEndpoints(shardedDONs, connMgrs) - require.Error(t, err) - }) - - t.Run("rejects nil connection manager", func(t *testing.T) { - t.Parallel() + connMgrs := [][]handlers.DON{{handlermocks.NewDON(t), handlermocks.NewDON(t)}} + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) - shardedDONs := []config.ShardedDONConfig{ - {DonName: "donA", Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1")}}}}, - } - connMgrs := [][]handlersDON{{nil}} - _, _, err := buildShardEndpoints(shardedDONs, connMgrs) + _, _, err = dons.BuildShardEndpoints() require.Error(t, err) + require.Contains(t, err.Error(), "disjoint") }) - t.Run("rejects duplicate node address across shards", func(t *testing.T) { + t.Run("AllMembers unions all shards", func(t *testing.T) { t.Parallel() + don0, don1 := handlermocks.NewDON(t), handlermocks.NewDON(t) shardedDONs := []config.ShardedDONConfig{ {DonName: "donA", Shards: []config.Shard{ - {Nodes: []config.NodeConfig{node("n1")}}, - {Nodes: []config.NodeConfig{node("n1")}}, // duplicate + {Nodes: []config.NodeConfig{node("n1"), node("n2")}}, + {Nodes: []config.NodeConfig{node("n3")}}, }}, } - connMgrs := [][]handlersDON{{handlermocks.NewDON(t), handlermocks.NewDON(t)}} - _, _, err := buildShardEndpoints(shardedDONs, connMgrs) - require.Error(t, err) - require.Contains(t, err.Error(), "disjoint") - }) - - t.Run("allMembers unions all shards", func(t *testing.T) { - t.Parallel() + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{don0, don1}}) + require.NoError(t, err) - eps := []*shardEndpoint{ - {members: []config.NodeConfig{node("n1"), node("n2")}}, - {members: []config.NodeConfig{node("n3")}}, - } - members := allMembers(eps) - require.Len(t, members, 3) + eps, _, err := dons.BuildShardEndpoints() + require.NoError(t, err) + require.Len(t, handlers.AllMembers(eps), 3) }) } From eb377b056daaddd9c80cc8e1e2a4f6c8852f4834 Mon Sep 17 00:00:00 2001 From: Bruno Moura Date: Tue, 29 Sep 2026 16:12:43 +0000 Subject: [PATCH 2/8] llo: v31 test coverage, plugin version naming, and plugin config wiring (#23850) * llo/datasource: fix flaky atomic cache write test * llo: fix integration tests for aggregation fault tolerance * llo: v31 aggregation fault tolerance integration tests * llo: blob catch-up integration tests * llo/bench: v31 blob broadcast and fetch failure paths tests * llo: observation compression tests with every report format * llo: v31 oracle restart tests * llo: assert attested retirement report at handover tests * llo: drop stale test stubs * llo: rename OCR31 selector to V31 Align the delegate config, test helpers and job plugin config key with the plugin version naming: OCR31 -> V31, IsOCR31 -> IsV31, WithOCR31 -> WithV31, and the TOML key ocrVersion = "3.1" -> pluginVersion = "v31". * bump chainlink-data-streams * llo: wire v31 plugin config knobs through The v31 knobs parsed and validated in pluginconfig.V31Config never reached the plugin factory, so every job ran on the plugin defaults. --- core/scripts/go.mod | 2 +- core/scripts/go.sum | 4 +- core/services/llo/delegate.go | 59 +- core/services/llo/delegate_test.go | 96 +++ .../llo/observation/data_source_test.go | 28 +- core/services/ocr2/delegate.go | 7 +- .../plugins/llo/bench/blob_failure_test.go | 249 ++++++ .../ocr2/plugins/llo/bench/harness_test.go | 57 +- .../services/ocr2/plugins/llo/helpers_test.go | 57 +- .../llo/history_backfill_integration_test.go | 17 +- .../ocr2/plugins/llo/integration_test.go | 787 ++++++++++++++++-- deployment/go.mod | 2 +- deployment/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- plugins/plugins.public.yaml | 2 +- system-tests/lib/go.mod | 2 +- system-tests/lib/go.sum | 4 +- system-tests/tests/go.mod | 2 +- system-tests/tests/go.sum | 4 +- 24 files changed, 1236 insertions(+), 165 deletions(-) create mode 100644 core/services/llo/delegate_test.go create mode 100644 core/services/ocr2/plugins/llo/bench/blob_failure_test.go diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 04d3b87f701..ee94f14be06 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -508,7 +508,7 @@ require ( github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 913f3b39798..526acfb1ff3 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1691,8 +1691,8 @@ github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.202609151 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= diff --git a/core/services/llo/delegate.go b/core/services/llo/delegate.go index 14daafc4f53..fdb78c89df0 100644 --- a/core/services/llo/delegate.go +++ b/core/services/llo/delegate.go @@ -22,6 +22,7 @@ import ( llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" llodatasource "github.com/smartcontractkit/chainlink-data-streams/llo/datasource" llov31 "github.com/smartcontractkit/chainlink-data-streams/llo/dev/v31" + lloconfig "github.com/smartcontractkit/chainlink-data-streams/llo/pluginconfig" lloprotocol "github.com/smartcontractkit/chainlink-data-streams/llo/protocol" "github.com/smartcontractkit/chainlink-data-streams/llo/retirement" "github.com/smartcontractkit/chainlink-data-streams/llo/transmitter" @@ -95,14 +96,17 @@ type DelegateConfig struct { LocalConfig ocr2types.LocalConfig NewOCR3DB func(pluginID int32) ocr3types.Database - // OCR3.1 (only required when OCR31 is true; see chainlink-data-streams - // llo/config.PluginConfig.OCRVersion) - OCR31 bool + // V31 runs the job on the v31 plugin, on libocr OCR3.1 (see + // chainlink-data-streams llo/pluginconfig.PluginConfig.PluginVersion) + V31 bool + // V31Config carries the v31 plugin knobs from the job's plugin config. Only + // read when V31 is true; zero fields fall through to the plugin defaults. + V31Config lloconfig.V31Config // BinaryNetworkEndpoint2Factory is the OCR3.1 ("2") network endpoint factory - // (peerWrapper.Peer3_1). Required when OCR31 is true. + // (peerWrapper.Peer3_1). Required when V31 is true. BinaryNetworkEndpoint2Factory ocr2types.BinaryNetworkEndpoint2Factory // KeyValueDatabaseFactory provides the replicated per-configDigest key-value - // store the OCR3.1 protocol requires. Required when OCR31 is true. + // store the OCR3.1 protocol requires. Required when V31 is true. KeyValueDatabaseFactory ocr3_1types.KeyValueDatabaseFactory } @@ -123,7 +127,7 @@ func NewDelegate(cfg DelegateConfig) (job.ServiceCtx, error) { if cfg.ShouldRetireCache == nil { return nil, errors.New("ShouldRetireCache must not be nil") } - if cfg.OCR31 { + if cfg.V31 { if cfg.KeyValueDatabaseFactory == nil { return nil, errors.New("KeyValueDatabaseFactory must not be nil when running OCR3.1") } @@ -190,7 +194,7 @@ func (d *delegate) Start(ctx context.Context) error { var oracle ocr2plus.Oracle var err error - if d.cfg.OCR31 { + if d.cfg.V31 { oracle, err = d.newOracleV31(i, configTracker, lggr, ocrLogger, psrrc) } else { oracle, err = d.newOracleV30(i, configTracker, lggr, ocrLogger, psrrc) @@ -248,25 +252,38 @@ func (d *delegate) newOracleV30(i int, configTracker ocr2types.ContractConfigTra }) } +// v31FactoryParams assembles the v31 plugin factory params, mapping the job's +// V31Config knobs onto it. Knobs left at zero are forwarded as zero, which the +// factory reads as "apply the plugin default". +func (d *delegate) v31FactoryParams(lggr logger.Logger, psrrc lloprotocol.PredecessorRetirementReportCache) llov31.PluginFactoryParams { + return llov31.PluginFactoryParams{ + VerboseLogging: d.cfg.ReportingPluginConfig.VerboseLogging || d.cfg.V31Config.VerboseLogging, + PredecessorRetirementReportCache: psrrc, + ShouldRetireCache: d.src, + RetirementReportCodec: d.cfg.RetirementReportCodec, + ChannelDefinitionCache: d.cfg.ChannelDefinitionCache, + DataSource: d.ds, + Logger: logger.Named(lggr, "ReportingPlugin"), + OnchainConfigCodec: lloprotocol.EVMOnchainConfigCodec{}, + ReportCodecs: d.reportCodecs, + OutcomeTelemetryCh: d.telem.GetOutcomeTelemetryCh(), + ReportTelemetryCh: d.telem.GetReportTelemetryCh(), + DonID: d.cfg.DonID, + MaxSnapshotRounds: d.cfg.V31Config.MaxSnapshotRounds, + BlobLifetimeRounds: d.cfg.V31Config.BlobLifetimeRounds, + MaxDurationBlobObservation: d.cfg.V31Config.MaxDurationBlobObservation.Duration(), + BlobInFlightWaitFactor: d.cfg.V31Config.BlobInFlightWaitFactor, + MaxBlobSnapshotAge: d.cfg.V31Config.MaxBlobSnapshotAge.Duration(), + MaxRoundPeriod: d.cfg.V31Config.MaxRoundPeriod.Duration(), + } +} + // newOracleV31 builds an OCR3.1 oracle running the llo/v31 reporting plugin. It // differs from v30 by the OCR3.1 oracle args (OCR3_1OracleArgs2), the "2" // network endpoint factory, and the required replicated KeyValueDatabaseFactory. func (d *delegate) newOracleV31(i int, configTracker ocr2types.ContractConfigTracker, lggr logger.Logger, ocrLogger ocrcommontypes.Logger, psrrc lloprotocol.PredecessorRetirementReportCache) (ocr2plus.Oracle, error) { factory := promwrapper31.NewReportingPluginFactory( - llov31.NewPluginFactory(llov31.PluginFactoryParams{ - Config: llov31.Config{VerboseLogging: d.cfg.ReportingPluginConfig.VerboseLogging}, - PredecessorRetirementReportCache: psrrc, - ShouldRetireCache: d.src, - RetirementReportCodec: d.cfg.RetirementReportCodec, - ChannelDefinitionCache: d.cfg.ChannelDefinitionCache, - DataSource: d.ds, - Logger: logger.Named(lggr, "ReportingPlugin"), - OnchainConfigCodec: lloprotocol.EVMOnchainConfigCodec{}, - ReportCodecs: d.reportCodecs, - OutcomeTelemetryCh: d.telem.GetOutcomeTelemetryCh(), - ReportTelemetryCh: d.telem.GetReportTelemetryCh(), - DonID: d.cfg.DonID, - }), + llov31.NewPluginFactory(d.v31FactoryParams(lggr, psrrc)), lggr, "", d.cfg.ChainID, diff --git a/core/services/llo/delegate_test.go b/core/services/llo/delegate_test.go new file mode 100644 index 00000000000..c54bd158820 --- /dev/null +++ b/core/services/llo/delegate_test.go @@ -0,0 +1,96 @@ +package llo + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + lloconfig "github.com/smartcontractkit/chainlink-data-streams/llo/pluginconfig" + llov30 "github.com/smartcontractkit/chainlink-data-streams/llo/v30" + "github.com/smartcontractkit/chainlink/v2/core/services/llo/telem" +) + +// newTestDelegate returns a delegate carrying only what v31FactoryParams reads. +func newTestDelegate(cfg DelegateConfig) *delegate { + return &delegate{cfg: cfg, telem: telem.NewTelemeterService(telem.TelemeterParams{})} +} + +func Test_delegate_v31FactoryParams(t *testing.T) { + t.Parallel() + + t.Run("forwards the V31Config knobs", func(t *testing.T) { + t.Parallel() + + params := newTestDelegate(DelegateConfig{ + DonID: 42, + V31Config: lloconfig.V31Config{ + MaxSnapshotRounds: 7, + BlobLifetimeRounds: 11, + MaxDurationBlobObservation: lloconfig.Duration(3 * time.Second), + BlobInFlightWaitFactor: 5, + MaxBlobSnapshotAge: lloconfig.Duration(9 * time.Second), + MaxRoundPeriod: lloconfig.Duration(time.Minute), + }, + }).v31FactoryParams(logger.Test(t), nil) + + assert.Equal(t, uint64(7), params.MaxSnapshotRounds) + assert.Equal(t, uint64(11), params.BlobLifetimeRounds) + assert.Equal(t, 3*time.Second, params.MaxDurationBlobObservation) + assert.Equal(t, uint64(5), params.BlobInFlightWaitFactor) + assert.Equal(t, 9*time.Second, params.MaxBlobSnapshotAge) + assert.Equal(t, time.Minute, params.MaxRoundPeriod) + assert.Equal(t, uint32(42), params.DonID) + }) + + t.Run("leaves unset knobs zero so the plugin applies its defaults", func(t *testing.T) { + t.Parallel() + + params := newTestDelegate(DelegateConfig{}).v31FactoryParams(logger.Test(t), nil) + + assert.Zero(t, params.MaxSnapshotRounds) + assert.Zero(t, params.BlobLifetimeRounds) + assert.Zero(t, params.MaxDurationBlobObservation) + assert.Zero(t, params.BlobInFlightWaitFactor) + assert.Zero(t, params.MaxBlobSnapshotAge) + assert.Zero(t, params.MaxRoundPeriod) + }) + + t.Run("forwards a negative MaxBlobSnapshotAge, which disables the age check", func(t *testing.T) { + t.Parallel() + + params := newTestDelegate(DelegateConfig{ + V31Config: lloconfig.V31Config{MaxBlobSnapshotAge: lloconfig.Duration(-time.Second)}, + }).v31FactoryParams(logger.Test(t), nil) + + assert.Equal(t, -time.Second, params.MaxBlobSnapshotAge) + }) + + t.Run("enables VerboseLogging from either the node or the job knob", func(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + node bool + job bool + want bool + }{ + {name: "neither", node: false, job: false, want: false}, + {name: "node only", node: true, job: false, want: true}, + {name: "job only", node: false, job: true, want: true}, + {name: "both", node: true, job: true, want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + params := newTestDelegate(DelegateConfig{ + ReportingPluginConfig: llov30.Config{VerboseLogging: tc.node}, + V31Config: lloconfig.V31Config{VerboseLogging: tc.job}, + }).v31FactoryParams(logger.Test(t), nil) + + assert.Equal(t, tc.want, params.VerboseLogging) + }) + } + }) +} diff --git a/core/services/llo/observation/data_source_test.go b/core/services/llo/observation/data_source_test.go index c565aa11cfc..1dc2b2db5a5 100644 --- a/core/services/llo/observation/data_source_test.go +++ b/core/services/llo/observation/data_source_test.go @@ -35,6 +35,7 @@ import ( ) type mockPipeline struct { + mu sync.Mutex run *pipeline.Run trrs pipeline.TaskRunResults err error @@ -46,9 +47,19 @@ type mockPipeline struct { func (m *mockPipeline) Run(ctx context.Context) (*pipeline.Run, pipeline.TaskRunResults, error) { m.runCount.Add(1) + m.mu.Lock() + defer m.mu.Unlock() return m.run, m.trrs, m.err } +// setResults swaps the results returned by Run. Keeping one pipeline pointer lets a test +// change generations without a concurrent observation round mixing results from two pointers. +func (m *mockPipeline) setResults(run *pipeline.Run, trrs pipeline.TaskRunResults, err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.run, m.trrs, m.err = run, trrs, err +} + func (m *mockPipeline) StreamIDs() []streams.StreamID { return m.streamIDs } @@ -593,13 +604,16 @@ func Test_DataSource(t *testing.T) { // Explicitly abort Cycle 1's background observation task before mutating pipelines cancel() - // Fix the pipeline with distinct values so we can verify generation + // Ignore the writes recorded so far; only cycle 2 writes are under test + mc.mu.Lock() + cycle2Start := len(mc.addCalls) + mc.mu.Unlock() + + // Fix the results with distinct values so we can verify generation. Swap them on the + // same pipeline pointer: replacing the registry entries would let a concurrent + // observation round read stream 1 from the old pipeline and streams 2, 3 from the new one. fixedPipeline := makePipelineWithMultipleStreamResults(sids, []any{decimal.NewFromFloat(111.0), decimal.NewFromFloat(222.0), decimal.NewFromFloat(333.0)}) - reg.mu.Lock() - reg.pipelines[1] = fixedPipeline - reg.pipelines[2] = fixedPipeline - reg.pipelines[3] = fixedPipeline - reg.mu.Unlock() + partialPipeline.setResults(fixedPipeline.run, fixedPipeline.trrs, fixedPipeline.err) time.Sleep(observationTimeout * 3) @@ -622,7 +636,7 @@ func Test_DataSource(t *testing.T) { defer mc.mu.Unlock() foundAtomicWrite := false - for _, call := range mc.addCalls { + for _, call := range mc.addCalls[cycle2Start:] { v1, has1 := call.values[llotypes.StreamID(1)] v2, has2 := call.values[llotypes.StreamID(2)] v3, has3 := call.values[llotypes.StreamID(3)] diff --git a/core/services/ocr2/delegate.go b/core/services/ocr2/delegate.go index b2955e4b89a..82f6ee4e327 100644 --- a/core/services/ocr2/delegate.go +++ b/core/services/ocr2/delegate.go @@ -1730,13 +1730,14 @@ func (d *Delegate) newServicesLLO( return NewDB(d.ds, spec.ID, pluginID, lggr) }, - OCR31: pluginCfg.IsOCR31(), + V31: pluginCfg.IsV31(), + V31Config: pluginCfg.V31, } - // OCR3.1 (llo/v31) additionally requires the "2" network endpoint factory and + // The v31 plugin additionally requires the "2" network endpoint factory and // a persistent replicated key-value store, wired here the same way the vault // and DKG OCR3.1 plugins are (pebble under OCR2().KeyValueStoreRootDir()). - if pluginCfg.IsOCR31() { + if pluginCfg.IsV31() { fullPath := filepath.Join(d.cfg.OCR2().KeyValueStoreRootDir(), jb.ExternalJobID.String()) if err = utils.EnsureDirAndMaxPerms(fullPath, os.FileMode(0o700)); err != nil { return nil, fmt.Errorf("failed to create LLO key value store directory: %w", err) diff --git a/core/services/ocr2/plugins/llo/bench/blob_failure_test.go b/core/services/ocr2/plugins/llo/bench/blob_failure_test.go new file mode 100644 index 00000000000..9f981e86370 --- /dev/null +++ b/core/services/ocr2/plugins/llo/bench/blob_failure_test.go @@ -0,0 +1,249 @@ +package bench + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3_1types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + llov31 "github.com/smartcontractkit/chainlink-data-streams/llo/dev/v31" + "github.com/smartcontractkit/chainlink-data-streams/llo/dev/v31/llotest" +) + +// The v31 blob path carries every stream value: the pump broadcasts a blob of +// observed values, and each round fetches the blobs the observations reference. +// These tests drive the plugin directly (as TestParity does) because neither +// end of that path can be made to fail from the node-level integration tests: +// the blob knobs are not plumbed through the LLO job spec, and the real +// transport does not drop blobs on demand. + +const ( + blobFailureN = 4 + blobFailureF = 1 +) + +// blobFailureWorkload is deliberately tiny: these tests assert on whether +// reports appear at all, not on throughput. +var blobFailureWorkload = workload{numChannels: 2, streamsPerChannel: 1} + +// fetchController wraps the in-memory broadcaster/fetcher so blob fetches can +// be failed on demand, modeling a blob that expired or that no peer served +// before the fetch deadline. +type fetchController struct { + *llotest.BlobBroadcastFetcher + failing atomic.Bool + attempts atomic.Uint64 +} + +func newFetchController() *fetchController { + return &fetchController{BlobBroadcastFetcher: llotest.NewBlobBroadcastFetcher()} +} + +func (f *fetchController) FetchBlob(ctx context.Context, handle ocr3_1types.BlobHandle) ([]byte, error) { + f.attempts.Add(1) + if f.failing.Load() { + return nil, errors.New("blob unavailable") + } + return f.BlobBroadcastFetcher.FetchBlob(ctx, handle) +} + +var _ ocr3_1types.BlobBroadcastFetcher = &fetchController{} + +// newV31 builds a v31 plugin over the given fetcher, applying opts to the +// factory params. +func newV31(tb testing.TB, defs llotypes.ChannelDefinitions, bbf ocr3_1types.BlobBroadcastFetcher, opts ...func(*llov31.PluginFactoryParams)) (ocr3_1types.ReportingPlugin[llotypes.ReportInfo], ocr3_1types.KeyValueDatabase) { + tb.Helper() + p, _, err := newV31Factory(defs, opts...).NewReportingPlugin(context.Background(), pluginConfig(blobFailureN, blobFailureF), bbf) + require.NoError(tb, err) + return p, newKVDB(tb) +} + +// driveRounds drives rounds from startSeq and returns the first seqNr that +// produced reports for every channel, or 0 if none did within the budget. It +// waits for the blob pump between rounds exactly as the benchmark drivers do. +func driveRounds(tb testing.TB, p ocr3_1types.ReportingPlugin[llotypes.ReportInfo], db ocr3_1types.KeyValueDatabase, bbf ocr3_1types.BlobBroadcastFetcher, pump *llotest.BlobBroadcastFetcher, startSeq uint64, rounds int, channels int) (reportingSeq uint64) { + tb.Helper() + for i := range rounds { + seqNr := startSeq + uint64(i) + before := pump.Broadcasts() + reports, _ := v31Round(tb, p, db, bbf, seqNr, blobFailureN) + if seqNr > 1 { + waitForPump(tb, pump, before) + } + if len(reports) >= channels { + return seqNr + } + } + return 0 +} + +// TestBlobBroadcastFailure covers the broadcast end of the blob path: while the +// pump cannot broadcast, no stream values reach the round, so no channel is +// reportable; the protocol must not fail, and reporting must resume once +// broadcasting works again. +func TestBlobBroadcastFailure(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + t.Parallel() + + defs, _ := blobFailureWorkload.channelDefinitions() + bbf := llotest.NewBlobBroadcastFetcher() + bbf.SetBroadcastError(errors.New("broadcast unavailable")) + p, db := newV31(t, defs, bbf) + + budget := int(warmupRounds(blobFailureWorkload.numChannels)) //nolint:gosec // G115: small round budget + seq := driveRounds(t, p, db, bbf, bbf, 1, budget, blobFailureWorkload.numChannels) + require.Zero(t, seq, "no channel should be reportable while blobs cannot be broadcast") + require.Positive(t, bbf.Broadcasts(), "the pump should have attempted to broadcast") + + bbf.SetBroadcastError(nil) + seq = driveRounds(t, p, db, bbf, bbf, uint64(budget+1), budget, blobFailureWorkload.numChannels) + require.NotZero(t, seq, "reporting should resume once blobs can be broadcast again") +} + +// v31RoundErr drives one v31 round like v31Round, but returns the +// StateTransition error instead of failing the test, so the abort path can be +// asserted on. On error nothing is committed, mirroring libocr discarding the +// transaction and retrying the same sequence number. +func v31RoundErr(tb testing.TB, p ocr3_1types.ReportingPlugin[llotypes.ReportInfo], db ocr3_1types.KeyValueDatabase, bbf ocr3_1types.BlobBroadcastFetcher, seqNr uint64) ([]ocr3types.ReportPlus[llotypes.ReportInfo], error) { + tb.Helper() + ctx := context.Background() + + var obs []byte + if seqNr > 1 { + rtx, err := db.NewReadTransaction() + require.NoError(tb, err) + obs, err = p.Observation(ctx, seqNr, ocrtypes.AttributedQuery{}, rtx, bbf) + rtx.Discard() + require.NoError(tb, err) + } + aos := bootOrReplicate(obs, blobFailureN, seqNr) + + wtx, err := db.NewReadWriteTransaction() + require.NoError(tb, err) + prec, err := p.StateTransition(ctx, seqNr, ocrtypes.AttributedQuery{}, aos, wtx, bbf) + if err != nil { + wtx.Discard() + return nil, err + } + require.NoError(tb, wtx.Commit()) + + reports, err := p.Reports(ctx, seqNr, prec) + require.NoError(tb, err) + return reports, nil +} + +// TestBlobFetchFailure covers the fetch end: the blobs are broadcast, but the +// round cannot retrieve them (expired, or no peer served them in time). A +// fetch failure is node-local, so dropping the observation would make the state +// transition non-deterministic across oracles; the round must abort instead, +// and reporting must resume once fetches succeed. +func TestBlobFetchFailure(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + t.Parallel() + + defs, _ := blobFailureWorkload.channelDefinitions() + fc := newFetchController() + fc.failing.Store(true) + p, db := newV31(t, defs, fc) + + budget := int(warmupRounds(blobFailureWorkload.numChannels)) //nolint:gosec // G115: small round budget + + // Drive until a round references a blob and therefore aborts. Earlier + // rounds only carry channel definitions, which travel inline. + var seqNr uint64 = 1 + var abortErr error + for range budget { + reports, err := v31RoundErr(t, p, db, fc, seqNr) + if err != nil { + abortErr = err + break + } + require.Empty(t, reports, "no channel should be reportable while blobs cannot be fetched") + before := fc.Broadcasts() + seqNr++ + waitForPump(t, fc.BlobBroadcastFetcher, before) + } + require.ErrorContains(t, abortErr, "failed to fetch blob for observation", + "a round referencing an unfetchable blob should abort") + require.Positive(t, fc.attempts.Load(), "the round should have attempted to fetch the blobs it referenced") + + // The aborted round committed nothing, so the same sequence number is + // retried, which is what libocr does. + fc.failing.Store(false) + seq := driveRounds(t, p, db, fc, fc.BlobBroadcastFetcher, seqNr, budget, blobFailureWorkload.numChannels) + require.NotZero(t, seq, "reporting should resume once blobs can be fetched again") +} + +// TestBlobRoundBounds covers the blob round knobs the factory validates: a blob +// lifetime that does not outlive the snapshots referencing it is rejected, as +// is one past the cap, while the tightest valid pairing still reports. +func TestBlobRoundBounds(t *testing.T) { + t.Parallel() + + // The tightest valid pairing: a snapshot gathered for one sequence number + // is consumed by a later one, so the blob must stay fetchable for + // BlobFetchMarginRounds past MaxSnapshotRounds. + const tightSnapshotRounds = 2 + const tightLifetimeRounds = tightSnapshotRounds + llov31.BlobFetchMarginRounds - 1 + + tcs := []struct { + name string + maxSnapshotRounds uint64 + blobLifetime uint64 + wantErr string + }{ + { + name: "lifetime leaving no fetch margin is rejected", + maxSnapshotRounds: 4, + blobLifetime: 4, + wantErr: "leaves less than", + }, + { + name: "lifetime past the cap is rejected", + blobLifetime: llov31.MaxBlobLifetimeRounds + 1, + wantErr: "exceeds MaxBlobLifetimeRounds", + }, + { + name: "tightest valid pairing reports", + maxSnapshotRounds: tightSnapshotRounds, + blobLifetime: tightLifetimeRounds, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + defs, _ := blobFailureWorkload.channelDefinitions() + opt := func(params *llov31.PluginFactoryParams) { + params.MaxSnapshotRounds = tc.maxSnapshotRounds + params.BlobLifetimeRounds = tc.blobLifetime + } + bbf := llotest.NewBlobBroadcastFetcher() + + if tc.wantErr != "" { + _, _, err := newV31Factory(defs, opt).NewReportingPlugin(context.Background(), pluginConfig(blobFailureN, blobFailureF), bbf) + require.ErrorContains(t, err, tc.wantErr) + return + } + + if testing.Short() { + t.Skip("too slow for testing.Short") + } + p, db := newV31(t, defs, bbf, opt) + budget := int(warmupRounds(blobFailureWorkload.numChannels)) //nolint:gosec // G115: small round budget + seq := driveRounds(t, p, db, bbf, bbf, 1, budget, blobFailureWorkload.numChannels) + require.NotZero(t, seq, "the tightest valid blob round bounds should still report") + }) + } +} diff --git a/core/services/ocr2/plugins/llo/bench/harness_test.go b/core/services/ocr2/plugins/llo/bench/harness_test.go index 54022c3b16a..d4a2e049c21 100644 --- a/core/services/ocr2/plugins/llo/bench/harness_test.go +++ b/core/services/ocr2/plugins/llo/bench/harness_test.go @@ -183,10 +183,14 @@ func reportCodecs() map[llotypes.ReportFormat]lloprotocol.ReportCodec { // to whole seconds, which would prevent reporting within a single wall-clock // second and diverge from v31). The 1ns interval effectively reports every // round while keeping both plugins on identical reportability rules. -func benchOffchainConfig() []byte { +// aggregationFaultTolerance is set to consensus F, the maximum the v31 factory +// accepts, so both plugins aggregate over the same observation set. +func benchOffchainConfig(f int) []byte { + aft := uint32(f) //nolint:gosec // bench F is a small positive constant b, err := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: 1, + AggregationFaultTolerance: &aft, }.Encode() if err != nil { panic(err) @@ -200,14 +204,14 @@ func pluginConfig(n, f int) ocr3types.ReportingPluginConfig { N: n, F: f, MaxDurationObservation: maxDurationObservation, - OffchainConfig: benchOffchainConfig(), + OffchainConfig: benchOffchainConfig(f), } } func buildV30(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) ocr3types.ReportingPlugin[llotypes.ReportInfo] { tb.Helper() factory := llov30.NewPluginFactory(llov30.PluginFactoryParams{ - Config: llov30.Config{VerboseLogging: false}, + VerboseLogging: false, ShouldRetireCache: mockShouldRetireCache{}, RetirementReportCodec: lloprotocol.StandardRetirementReportCodec{}, ChannelDefinitionCache: &mockChannelDefinitionCache{defs: defs}, @@ -221,10 +225,11 @@ func buildV30(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) ocr3typ return p } -func buildV31(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) (ocr3_1types.ReportingPlugin[llotypes.ReportInfo], ocr3_1types.KeyValueDatabase, *llotest.BlobBroadcastFetcher) { - tb.Helper() - factory := llov31.NewPluginFactory(llov31.PluginFactoryParams{ - Config: llov31.Config{VerboseLogging: false}, +// newV31Factory builds the v31 plugin factory over the given definitions. opts +// mutate the factory params, e.g. to override the blob pump knobs. +func newV31Factory(defs llotypes.ChannelDefinitions, opts ...func(*llov31.PluginFactoryParams)) *llov31.PluginFactory { + params := llov31.PluginFactoryParams{ + VerboseLogging: false, ShouldRetireCache: mockShouldRetireCache{}, RetirementReportCodec: lloprotocol.StandardRetirementReportCodec{}, ChannelDefinitionCache: &mockChannelDefinitionCache{defs: defs}, @@ -232,26 +237,38 @@ func buildV31(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) (ocr3_1 Logger: logger.Nop(), OnchainConfigCodec: mockOnchainConfigCodec{}, ReportCodecs: reportCodecs(), - }) + } + for _, opt := range opts { + opt(¶ms) + } + return llov31.NewPluginFactory(params) +} + +// newKVDB returns libocr's in-memory KeyValueDatabase (the same helper v31's +// integration tests use): a btree behind the production KeyValueDatabaseFactory +// interface, whose Commit applies to memory with no WAL/fsync. This isolates +// the plugin's CPU/allocation cost from storage-engine cost, so the v31 numbers +// are directly comparable to v30's in-memory Outcome blob. +func newKVDB(tb testing.TB) ocr3_1types.KeyValueDatabase { + tb.Helper() + dbFactory := memkvdb.NewStatelessInMemoryKeyValueDatabaseFactory() + db, err := dbFactory.NewKeyValueDatabase(benchConfigDigest) + require.NoError(tb, err) + tb.Cleanup(func() { _ = db.Close() }) + return db +} + +func buildV31(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) (ocr3_1types.ReportingPlugin[llotypes.ReportInfo], ocr3_1types.KeyValueDatabase, *llotest.BlobBroadcastFetcher) { + tb.Helper() // v31 disseminates stream values exclusively through blobs, so the plugin // needs a BlobBroadcastFetcher both at construction (the blob pump // broadcasts through it) and on every Observation/StateTransition call (the // round decodes observations by fetching the blobs they reference). A nil // fetcher yields a plugin whose observations never carry stream values. bbf := llotest.NewBlobBroadcastFetcher() - p, _, err := factory.NewReportingPlugin(context.Background(), pluginConfig(n, f), bbf) + p, _, err := newV31Factory(defs).NewReportingPlugin(context.Background(), pluginConfig(n, f), bbf) require.NoError(tb, err) - - // libocr's in-memory KeyValueDatabase (the same helper v31's integration - // tests use): a btree behind the production KeyValueDatabaseFactory - // interface, whose Commit applies to memory with no WAL/fsync. This isolates - // the plugin's CPU/allocation cost from storage-engine cost, so the v31 - // numbers are directly comparable to v30's in-memory Outcome blob. - dbFactory := memkvdb.NewStatelessInMemoryKeyValueDatabaseFactory() - db, err := dbFactory.NewKeyValueDatabase(benchConfigDigest) - require.NoError(tb, err) - tb.Cleanup(func() { _ = db.Close() }) - return p, db, bbf + return p, newKVDB(tb), bbf } // --------------------------------------------------------------------------- diff --git a/core/services/ocr2/plugins/llo/helpers_test.go b/core/services/ocr2/plugins/llo/helpers_test.go index 09fcc3234cf..eaaac68ec39 100644 --- a/core/services/ocr2/plugins/llo/helpers_test.go +++ b/core/services/ocr2/plugins/llo/helpers_test.go @@ -136,12 +136,13 @@ func (node *Node) DeleteJob(t *testing.T, id int32) { require.NoError(t, err) } -func (node *Node) AddLLOJob(t *testing.T, spec string) { +func (node *Node) AddLLOJob(t *testing.T, spec string) (id int32) { c := node.App.GetConfig() job, err := validate.ValidatedOracleSpecToml(t.Context(), c.OCR2(), c.Insecure(), spec, nil) require.NoError(t, err) err = node.App.AddJobV2(t.Context(), &job) require.NoError(t, err) + return job.ID } func (node *Node) AddBootstrapJob(t *testing.T, spec string) { @@ -159,6 +160,23 @@ func setupNode( csaKey csakey.KeyV2, f func(*chainlink.Config), ) (app chainlink.Application, peerID string, clientPubKey credentials.StaticSizedPublicKey, ocr2kb ocr2key.KeyBundle, observedLogs *observer.ObservedLogs) { + app, peerID, clientPubKey, ocr2kb, observedLogs, _ = setupRestartableNode(t, port, dbName, backend, csaKey, f) + return app, peerID, clientPubKey, ocr2kb, observedLogs +} + +// setupRestartableNode is setupNode plus a restart hook. restart stops the +// application, runs whileDown (if any), then starts a new one on the same +// config, database, keys and key-value store root, which is a process restart +// with every piece of persisted state left in place: jobs, and the OCR3.1 +// (llo/v31) pebble store. +func setupRestartableNode( + t *testing.T, + port int, + dbName string, + backend evmtypes.Backend, + csaKey csakey.KeyV2, + f func(*chainlink.Config), +) (app chainlink.Application, peerID string, clientPubKey credentials.StaticSizedPublicKey, ocr2kb ocr2key.KeyBundle, observedLogs *observer.ObservedLogs, restart func(t *testing.T, whileDown func()) chainlink.Application) { k := big.NewInt(int64(port)) // keys unique to port p2pKey := p2pkey.MustNewV2XXXTestingOnly(k) rdr := keystest.NewRandReaderFromSeed(int64(port)) @@ -221,19 +239,32 @@ func setupNode( }) lggr, observedLogs := logger.TestLoggerObserved(t, config.Log().Level()) - if backend != nil { - app = cltest.NewApplicationWithConfigV2OnSimulatedBlockchain(t, config, backend, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) - } else { - app = cltest.NewApplicationWithConfig(t, config, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) + // newApp builds and starts an application over the config above. Start + // registers its own cleanup, which tolerates an application this test + // already stopped. + newApp := func(t *testing.T) chainlink.Application { + var a chainlink.Application + if backend != nil { + a = cltest.NewApplicationWithConfigV2OnSimulatedBlockchain(t, config, backend, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) + } else { + a = cltest.NewApplicationWithConfig(t, config, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) + } + require.NoError(t, a.Start(t.Context())) + return a } - err := app.Start(t.Context()) - require.NoError(t, err) - t.Cleanup(func() { - assert.NoError(t, app.Stop()) - }) + app = newApp(t) + current := app + restart = func(t *testing.T, whileDown func()) chainlink.Application { + require.NoError(t, current.Stop()) + if whileDown != nil { + whileDown() + } + current = newApp(t) + return current + } - return app, p2pKey.PeerID().Raw(), csaKey.StaticSizedPublicKey(), ocr2kb, observedLogs + return app, p2pKey.PeerID().Raw(), csaKey.StaticSizedPublicKey(), ocr2kb, observedLogs, restart } // receiveWithTimeout receives from the packet channel with a timeout. @@ -368,8 +399,8 @@ func addLLOJob( pluginConfig, relayType, relayConfig string, -) { - node.AddLLOJob(t, fmt.Sprintf(` +) (id int32) { + return node.AddLLOJob(t, fmt.Sprintf(` type = "offchainreporting2" schemaVersion = 1 name = "%s" diff --git a/core/services/ocr2/plugins/llo/history_backfill_integration_test.go b/core/services/ocr2/plugins/llo/history_backfill_integration_test.go index a5b150e5178..c0dc999bd4c 100644 --- a/core/services/ocr2/plugins/llo/history_backfill_integration_test.go +++ b/core/services/ocr2/plugins/llo/history_backfill_integration_test.go @@ -104,19 +104,19 @@ func quoteBackfillString(benchmark float64) string { func TestIntegration_LLO_history_backfill(t *testing.T) { t.Parallel() - for _, ocr31 := range []bool{false, true} { + for _, v31 := range []bool{false, true} { name := "OCR3.0/v30" - if ocr31 { + if v31 { name = "OCR3.1/v31" } t.Run(name, func(t *testing.T) { t.Parallel() - testIntegrationLLOHistoryBackfill(t, ocr31) + testIntegrationLLOHistoryBackfill(t, v31) }) } } -func testIntegrationLLOHistoryBackfill(t *testing.T, ocr31 bool) { +func testIntegrationLLOHistoryBackfill(t *testing.T, v31 bool) { const ( salt = 600 donID = uint32(776655) @@ -134,6 +134,7 @@ func testIntegrationLLOHistoryBackfill(t *testing.T, ocr31 bool) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: true, } @@ -178,8 +179,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } nativeStrm := Stream{ @@ -223,8 +224,8 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi backend.Commit() productionConfigOpts := []OCRConfigOption{WithOracles(oracles), WithOffchainConfig(offchainConfig)} - if ocr31 { - productionConfigOpts = append(productionConfigOpts, WithOCR31()) + if v31 { + productionConfigOpts = append(productionConfigOpts, WithV31()) } setProductionConfig( t, donID, steve, backend, configurator, configuratorAddress, nodes, diff --git a/core/services/ocr2/plugins/llo/integration_test.go b/core/services/ocr2/plugins/llo/integration_test.go index 82bf18a025c..a1255e055da 100644 --- a/core/services/ocr2/plugins/llo/integration_test.go +++ b/core/services/ocr2/plugins/llo/integration_test.go @@ -73,6 +73,10 @@ var ( fNodes = uint8(1) nNodes = 4 // number of nodes (not including bootstrap) reportTimeout = time.Second * 60 + + // v31 requires the contribution floor to be set explicitly. Match consensus + // F so aggregation uses the full observation set. + aggregationFaultTolerance = func() *uint32 { v := uint32(fNodes); return &v }() ) func setupBlockchain(t *testing.T, adders ...*bind.TransactOpts) ( @@ -227,11 +231,11 @@ type OCRConfig struct { F int OnchainConfig []byte - // ocr31 selects the OCR3.1 (llo/v31) config format when true. It changes the + // v31 selects the OCR3.1 config format the v31 plugin runs on. It changes the // confighelper used by generateConfig (ocr3_1confighelper => offchainConfigVersion // 310) so the on-chain config matches what an OCR3.1 node validates. The // config digest prefix (LLO 0x0009) is identical across OCR3.0/3.1. - ocr31 bool + v31 bool } func makeDefaultOCRConfig() *OCRConfig { @@ -292,10 +296,10 @@ func WithOracles(oracles []confighelper.OracleIdentityExtra) OCRConfigOption { } } -// WithOCR31 switches config generation to the OCR3.1 (llo/v31) format. -func WithOCR31() OCRConfigOption { +// WithV31 switches config generation to the OCR3.1 format the v31 plugin runs on. +func WithV31() OCRConfigOption { return func(cfg *OCRConfig) { - cfg.ocr31 = true + cfg.v31 = true } } @@ -309,7 +313,7 @@ func generateConfig(t *testing.T, opts ...OCRConfigOption) (signers []types.Onch } t.Logf("Using OCR config: %+v\n", cfg) var err error - if cfg.ocr31 { + if cfg.v31 { signers, transmitters, f, outOnchainConfig, offchainConfigVersion, offchainConfig, err = generateOCR31Config(cfg) } else { signers, transmitters, f, outOnchainConfig, offchainConfigVersion, offchainConfig, err = ocr3confighelper.ContractSetConfigArgsForTests( @@ -483,22 +487,54 @@ func promoteStagingConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, backend.Commit() } -func TestIntegration_LLO_evm_premium_legacy(t *testing.T) { - t.Parallel() - offchainConfigs := []lloprotocol.OffchainConfig{ +// offchainConfigCase is a named offchain config for a test matrix. The name is +// spelled out rather than derived from the struct so subtest names stay stable +// (the struct holds a pointer, whose formatted address changes per run). +type offchainConfigCase struct { + name string + cfg lloprotocol.OffchainConfig +} + +// offchainConfigCases is the offchain config matrix shared by the tests that +// cover every report format: both protocol versions, plus observation +// compression. Compression is a v30 codec setting; v31 always frames its blobs +// with zstd and ignores the flag. +func offchainConfigCases() []offchainConfigCase { + return []offchainConfigCase{ + { + name: "protocolVersion=0", + cfg: lloprotocol.OffchainConfig{ + ProtocolVersion: 0, + DefaultMinReportIntervalNanoseconds: 0, + AggregationFaultTolerance: aggregationFaultTolerance, + }, + }, { - ProtocolVersion: 0, - DefaultMinReportIntervalNanoseconds: 0, + name: "protocolVersion=1", + cfg: lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: 1, + AggregationFaultTolerance: aggregationFaultTolerance, + }, }, { - ProtocolVersion: 1, - DefaultMinReportIntervalNanoseconds: 1, + name: "protocolVersion=1/compressedObservations", + cfg: lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: 1, + AggregationFaultTolerance: aggregationFaultTolerance, + EnableObservationCompression: true, + }, }, } - for _, offchainConfig := range offchainConfigs { - t.Run(fmt.Sprintf("offchainConfig=%+v", offchainConfig), func(t *testing.T) { +} + +func TestIntegration_LLO_evm_premium_legacy(t *testing.T) { + t.Parallel() + for _, oc := range offchainConfigCases() { + t.Run(oc.name, func(t *testing.T) { t.Parallel() - testIntegrationLLOEVMPremiumLegacy(t, offchainConfig) + testIntegrationLLOEVMPremiumLegacy(t, oc.cfg) }) } } @@ -725,34 +761,24 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi func TestIntegration_LLO_multi_formats(t *testing.T) { t.Parallel() - offchainConfigs := []lloprotocol.OffchainConfig{ - { - ProtocolVersion: 0, - DefaultMinReportIntervalNanoseconds: 0, - }, - { - ProtocolVersion: 1, - DefaultMinReportIntervalNanoseconds: 1, - }, - } - ocrVersions := []struct { - name string - ocr31 bool + pluginVersions := []struct { + name string + v31 bool }{ {"OCR3.0/v30", false}, {"OCR3.1/v31", true}, } - for _, offchainConfig := range offchainConfigs { - for _, ov := range ocrVersions { - t.Run(fmt.Sprintf("%s/offchainConfig=%+v", ov.name, offchainConfig), func(t *testing.T) { + for _, oc := range offchainConfigCases() { + for _, ov := range pluginVersions { + t.Run(fmt.Sprintf("%s/%s", ov.name, oc.name), func(t *testing.T) { t.Parallel() - testIntegrationLLOMultiFormats(t, offchainConfig, ov.ocr31) + testIntegrationLLOMultiFormats(t, oc.cfg, ov.v31) }) } } } -func testIntegrationLLOMultiFormats(t *testing.T, offchainConfig lloprotocol.OffchainConfig, ocr31 bool) { +func testIntegrationLLOMultiFormats(t *testing.T, offchainConfig lloprotocol.OffchainConfig, v31 bool) { testStartTimeStamp := time.Now() expirationWindow := uint32(3600) @@ -1088,8 +1114,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } bridgeName := "superbridge" @@ -1279,8 +1305,8 @@ dp -> deribit_funding_interval_hours_parse -> deribit_funding_interval_hours_dec // Set config on configurator productionConfigOpts := []OCRConfigOption{WithOracles(oracles), WithOffchainConfig(offchainConfig)} - if ocr31 { - productionConfigOpts = append(productionConfigOpts, WithOCR31()) + if v31 { + productionConfigOpts = append(productionConfigOpts, WithV31()) } digest := setProductionConfig( t, donID, steve, backend, configurator, configuratorAddress, nodes, productionConfigOpts..., @@ -1523,6 +1549,7 @@ func TestIntegration_LLO_stress_test_V1(t *testing.T) { WithOffchainConfig(lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(defaultMinReportInterval), + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: true, }), func(cfg *OCRConfig) { @@ -1759,6 +1786,7 @@ func TestIntegration_LLO_transmit_errors(t *testing.T) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(50 * time.Millisecond), + AggregationFaultTolerance: aggregationFaultTolerance, } clientCSAKeys := make([]csakey.KeyV2, nNodes) @@ -1907,6 +1935,47 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, serverPubKey }) } +// assertRetirementReport checks the attested retirement report every oracle +// persisted for a retired config. The successor instance reads this report to +// learn where the predecessor stopped, so its contents are what make the +// handover gapless: a wrong ValidAfterNanoseconds either replays or skips a +// window of time that no report covers. +// +// truncateToSecond covers protocol version 0, whose timestamps carry only +// second resolution. +func assertRetirementReport(t *testing.T, nodes []Node, digest ocr2types.ConfigDigest, wantProtocolVersion uint32, wantValidAfterNanoseconds map[llotypes.ChannelID]uint64, truncateToSecond bool) { + t.Helper() + + toSecond := func(m map[llotypes.ChannelID]uint64) map[llotypes.ChannelID]uint64 { + if !truncateToSecond { + return m + } + out := make(map[llotypes.ChannelID]uint64, len(m)) + for k, v := range m { + out[k] = v / 1e9 * 1e9 + } + return out + } + + for i, node := range nodes { + var raw []byte + require.Eventually(t, func() bool { + return node.App.GetDB().GetContext(t.Context(), &raw, + `SELECT attested_retirement_report FROM llo_retirement_report_cache WHERE config_digest = $1`, digest[:]) == nil + }, reportTimeout, 100*time.Millisecond, "oracle %d should persist an attested retirement report for the retired config", i) + + var attested lloprotocol.AttestedRetirementReport + require.NoError(t, proto.Unmarshal(raw, &attested)) + assert.NotZero(t, attested.SeqNr, "oracle %d: retirement report should name the sequence number it retired at", i) + assert.GreaterOrEqual(t, len(attested.Sigs), int(fNodes)+1, "oracle %d: an attested retirement report needs at least f+1 signatures", i) + + rr, err := (lloprotocol.StandardRetirementReportCodec{}).Decode(attested.RetirementReport) + require.NoError(t, err) + assert.Equal(t, wantProtocolVersion, rr.ProtocolVersion, "oracle %d", i) + assert.Equal(t, toSecond(wantValidAfterNanoseconds), toSecond(rr.ValidAfterNanoseconds), "oracle %d", i) + } +} + func TestIntegration_LLO_blue_green_lifecycle(t *testing.T) { t.Parallel() @@ -1915,25 +1984,26 @@ func TestIntegration_LLO_blue_green_lifecycle(t *testing.T) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 0, DefaultMinReportIntervalNanoseconds: 0, + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: false, } - for _, ocr31 := range []bool{false, true} { + for _, v31 := range []bool{false, true} { name := "OCR3.0/v30" - if ocr31 { + if v31 { name = "OCR3.1/v31" } t.Run(name, func(t *testing.T) { t.Parallel() - testIntegrationLLOBlueGreenLifecycle(t, offchainConfig, ocr31) + testIntegrationLLOBlueGreenLifecycle(t, offchainConfig, v31) }) } } -func testIntegrationLLOBlueGreenLifecycle(t *testing.T, offchainConfig lloprotocol.OffchainConfig, ocr31 bool) { - // withVersion appends WithOCR31() to config options when running the v31 variant. +func testIntegrationLLOBlueGreenLifecycle(t *testing.T, offchainConfig lloprotocol.OffchainConfig, v31 bool) { + // withVersion appends WithV31() to config options when running the v31 variant. withVersion := func(opts ...OCRConfigOption) []OCRConfigOption { - if ocr31 { - return append(opts, WithOCR31()) + if v31 { + return append(opts, WithV31()) } return opts } @@ -2011,8 +2081,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } addOCRJobsEVMPremiumLegacy(t, streams, serverPubKey, serverURL, configuratorAddress, bootstrapPeerID, bootstrapNodePort, nodes, configStoreAddress, clientPubKeys, pluginConfig, relayType, relayConfig) @@ -2161,6 +2231,11 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi } assert.Less(t, initialPromotedGreenReport.ValidAfterNanoseconds, initialPromotedGreenReport.ObservationTimestampNanoseconds) + + // The retired instance's report is what green handed over from. + assertRetirementReport(t, nodes, blueDigest, offchainConfig.ProtocolVersion, + map[llotypes.ChannelID]uint64{1: finalBlueReport.ObservationTimestampNanoseconds}, + offchainConfig.ProtocolVersion == 0) } // retired instance does not produce reports { @@ -2237,6 +2312,11 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi finalGreenReport := allReports[greenDigest][len(allReports[greenDigest])-1] assert.Less(t, finalGreenReport.ValidAfterNanoseconds, finalGreenReport.ObservationTimestampNanoseconds) assert.Less(t, initialPromotedBlueReport.ValidAfterNanoseconds, initialPromotedBlueReport.ObservationTimestampNanoseconds) + + // Green ran on the config it was staged with, which is the one blue + // retired under; blue's new staging config is not yet in effect. + assertRetirementReport(t, nodes, greenDigest, 0, + map[llotypes.ChannelID]uint64{1: finalGreenReport.ObservationTimestampNanoseconds}, true) } // adding a new channel definition is picked up on the fly { @@ -2282,33 +2362,28 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi assert.Equal(t, "2976.39", r.Values[0].(*lloprotocol.Decimal).String()) } } - t.Run("deleting the jobs turns off oracles and cleans up resources", func(t *testing.T) { - t.Skip("TODO - MERC-3524") - }) - t.Run("adding new jobs again picks up the correct configs", func(t *testing.T) { - t.Skip("TODO - MERC-3524") - }) }) } func TestIntegration_LLO_channel_merging_owners_adders(t *testing.T) { t.Parallel() - for _, ocr31 := range []bool{false, true} { + for _, v31 := range []bool{false, true} { name := "OCR3.0/v30" - if ocr31 { + if v31 { name = "OCR3.1/v31" } t.Run(name, func(t *testing.T) { t.Parallel() - testIntegrationLLOChannelMerging(t, ocr31) + testIntegrationLLOChannelMerging(t, v31) }) } } -func testIntegrationLLOChannelMerging(t *testing.T, ocr31 bool) { +func testIntegrationLLOChannelMerging(t *testing.T, v31 bool) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: true, } @@ -2387,8 +2462,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } // Add stream specs and LLO jobs to all nodes @@ -2410,8 +2485,8 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi // Set initial OCR config mergeConfigOpts := []OCRConfigOption{WithOracles(oracles), WithOffchainConfig(offchainConfig)} - if ocr31 { - mergeConfigOpts = append(mergeConfigOpts, WithOCR31()) + if v31 { + mergeConfigOpts = append(mergeConfigOpts, WithV31()) } digest := setProductionConfig( t, donID, steve, backend, configurator, configuratorAddress, nodes, mergeConfigOpts..., @@ -2818,19 +2893,19 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi // and no longer transmits reports for that channel. func TestIntegration_LLO_tombstone_stops_observations_and_reports(t *testing.T) { t.Parallel() - for _, ocr31 := range []bool{false, true} { + for _, v31 := range []bool{false, true} { name := "OCR3.0/v30" - if ocr31 { + if v31 { name = "OCR3.1/v31" } t.Run(name, func(t *testing.T) { t.Parallel() - testIntegrationLLOTombstone(t, ocr31) + testIntegrationLLOTombstone(t, v31) }) } } -func testIntegrationLLOTombstone(t *testing.T, ocr31 bool) { +func testIntegrationLLOTombstone(t *testing.T, v31 bool) { const ( salt = 500 donID = uint32(777666) @@ -2841,6 +2916,7 @@ func testIntegrationLLOTombstone(t *testing.T, ocr31 bool) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: true, } @@ -2885,8 +2961,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } var streamACalls, streamBCalls atomic.Uint64 @@ -2931,8 +3007,8 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi backend.Commit() tombstoneConfigOpts := []OCRConfigOption{WithOracles(oracles), WithOffchainConfig(offchainConfig)} - if ocr31 { - tombstoneConfigOpts = append(tombstoneConfigOpts, WithOCR31()) + if v31 { + tombstoneConfigOpts = append(tombstoneConfigOpts, WithV31()) } setProductionConfig( t, donID, steve, backend, configurator, configuratorAddress, nodes, @@ -3047,6 +3123,7 @@ func TestIntegration_LLO_bridgeConnManagerHappyPath(t *testing.T) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, } clientCSAKeys := make([]csakey.KeyV2, nNodes) @@ -3156,9 +3233,17 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi } func setupNodes(t *testing.T, nNodes int, backend evmtypes.Backend, clientCSAKeys []csakey.KeyV2, f func(*chainlink.Config)) (oracles []confighelper.OracleIdentityExtra, nodes []Node) { + oracles, nodes, _ = setupNodesWithRestart(t, nNodes, backend, clientCSAKeys, f) + return oracles, nodes +} + +// setupNodesWithRestart is setupNodes plus a per-node restart hook. See +// setupRestartableNode. +func setupNodesWithRestart(t *testing.T, nNodes int, backend evmtypes.Backend, clientCSAKeys []csakey.KeyV2, f func(*chainlink.Config)) (oracles []confighelper.OracleIdentityExtra, nodes []Node, restarts []func(t *testing.T, whileDown func()) chainlink.Application) { ports := freeport.GetN(t, nNodes) for i := range nNodes { - app, peerID, transmitter, kb, observedLogs := setupNode(t, ports[i], fmt.Sprintf("oracle_streams_%d", i), backend, clientCSAKeys[i], f) + app, peerID, transmitter, kb, observedLogs, restart := setupRestartableNode(t, ports[i], fmt.Sprintf("oracle_streams_%d", i), backend, clientCSAKeys[i], f) + restarts = append(restarts, restart) nodes = append(nodes, Node{ app, transmitter, kb, observedLogs, @@ -3175,7 +3260,7 @@ func setupNodes(t *testing.T, nNodes int, backend evmtypes.Backend, clientCSAKey ConfigEncryptionPublicKey: kb.ConfigEncryptionPublicKey(), }) } - return oracles, nodes + return oracles, nodes, restarts } func newChannelDefinitionsServer(t *testing.T, channelDefinitions llotypes.ChannelDefinitions) (url string, sha [32]byte) { @@ -3234,3 +3319,563 @@ func newSingleABIEncoder(typ string, multiplier *sqlutil.Big) (enc lloevm.ABIEnc } return enc } + +// TestIntegration_LLO_aggregation_fault_tolerance exercises the v31 contribution +// floor. aggregationFaultTolerance sets the fewest contributions an aggregate may +// be built from (2*aft+1), so a stream observed by too few oracles must not +// produce an aggregate and its channel must not report, while a stream observed +// by the whole DON keeps reporting. +// +// With nNodes=4 and F=1: aft=1 sets a floor of 3 contributions, aft=0 a floor of +// 1. A stream whose job exists on only 2 nodes therefore reports under aft=0 and +// is withheld under aft=1. +func TestIntegration_LLO_aggregation_fault_tolerance(t *testing.T) { + t.Parallel() + tcs := []struct { + name string + // aft is the aggregationFaultTolerance set in the offchain config. + aft uint32 + // partialReports is whether the channel whose stream only 2 of the 4 + // oracles observe is expected to produce reports. + partialReports bool + salt int64 + donID uint32 + }{ + {name: "floor of 3 withholds an under-observed stream", aft: 1, partialReports: false, salt: 700, donID: uint32(555111)}, + {name: "floor of 1 admits an under-observed stream", aft: 0, partialReports: true, salt: 750, donID: uint32(555222)}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testIntegrationLLOAggregationFaultTolerance(t, tc.aft, tc.partialReports, tc.salt, tc.donID) + }) + } +} + +func testIntegrationLLOAggregationFaultTolerance(t *testing.T, aft uint32, expectPartialReports bool, salt int64, donID uint32) { + const ( + // streamIDFull has a stream job on every oracle. + streamIDFull = uint32(690) + // streamIDPartial has a stream job on only the first two oracles, so it + // gathers 2 contributions per round. + streamIDPartial = uint32(691) + // nPartialObservers is how many oracles observe streamIDPartial. + nPartialObservers = 2 + + channelFull = uint32(1) + channelPartial = uint32(2) + ) + + offchainConfig := lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: &aft, + } + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + for i := range nNodes { + k := big.NewInt(salt + int64(i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress, _, _, _, _ := setupBlockchain(t) + fromBlock := 1 + + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo_aft", backend, bootstrapCSAKey, nil) + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + packetCh := make(chan *packet, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, serverKey, packetCh) + serverURL := startMercuryServer(t, srv, clientPubKeys) + + oracles, nodes := setupNodes(t, nNodes, backend, clientCSAKeys, func(c *chainlink.Config) { + c.Mercury.Transmitter.Protocol = new(mercurytransmitter.MercuryTransmitterProtocolGRPC) + }) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, fmt.Sprintf("job-aft-%d", donID), relayType, relayConfig) + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d +pluginVersion = "v31"`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + + priceFull := decimal.NewFromFloat(111.1) + pricePartial := decimal.NewFromFloat(222.2) + for i, node := range nodes { + bridgeFull := createSingleDecimalBridge(t, "aft-full", i, priceFull, node.App.BridgeORM()) + addSingleDecimalStreamJob(t, node, streamIDFull, bridgeFull) + if i < nPartialObservers { + bridgePartial := createSingleDecimalBridge(t, "aft-partial", i, pricePartial, node.App.BridgeORM()) + addSingleDecimalStreamJob(t, node, streamIDPartial, bridgePartial) + } + addLLOJob( + t, + node, + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[i], + "aggregation-fault-tolerance-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + channelDefinitions := llotypes.ChannelDefinitions{ + channelFull: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + {StreamID: streamIDFull, Aggregator: llotypes.AggregatorMedian}, + }, + }, + channelPartial: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + {StreamID: streamIDPartial, Aggregator: llotypes.AggregatorMedian}, + }, + }, + } + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, + WithOracles(oracles), WithOffchainConfig(offchainConfig), WithV31(), + ) + + // collectChannels drains transmitted JSON reports for the given duration and + // returns the set of channel IDs seen. + collectChannels := func(d time.Duration) map[uint32]bool { + seen := make(map[uint32]bool) + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + pckt, errReceive := receiveWithTimeout(t, packetCh, time.Until(deadline)) + if errReceive != nil { + break + } + req := pckt.req + if req.ReportFormat != uint32(llotypes.ReportFormatJSON) { + continue + } + _, _, r, _, errDecode := (lloreportcodec.JSONReportCodec{}).UnpackDecode(req.Payload) + if errDecode != nil { + continue + } + seen[r.ChannelID] = true + } + return seen + } + + // The fully observed channel must report regardless of the floor. This also + // establishes that the DON has reached a reporting steady state before + // judging the under-observed channel. + require.Eventually(t, func() bool { + return collectChannels(2 * time.Second)[channelFull] + }, reportTimeout, 100*time.Millisecond, "fully observed channel should produce reports") + + // Now judge the under-observed channel over a window that spans many rounds. + seen := collectChannels(15 * time.Second) + require.True(t, seen[channelFull], "fully observed channel should keep reporting") + if expectPartialReports { + require.True(t, seen[channelPartial], + "under-observed channel should report when the contribution floor is met") + } else { + require.False(t, seen[channelPartial], + "under-observed channel should not report when it cannot meet the contribution floor") + } +} + +// TestIntegration_LLO_v31_blob_catchup covers the v31 blob path for an oracle +// that is not present for the earlier rounds. v31 disseminates stream values +// through blobs, so such an oracle can only contribute once it has fetched the +// blobs and state the running DON produced without it. +// +// Two ways in: an oracle whose LLO job is created after the DON is already +// reporting, and an oracle whose job is deleted mid-run and re-created after +// the DON has advanced many rounds without it. In both cases the remaining +// 3 oracles (2F+1) keep the DON reporting throughout. +func TestIntegration_LLO_v31_blob_catchup(t *testing.T) { + t.Parallel() + tcs := []struct { + name string + // rejoin starts the lagging oracle with an LLO job and deletes it + // mid-run, instead of never creating it until catch-up time. + rejoin bool + salt int64 + donID uint32 + }{ + {name: "oracle joining after the DON is reporting catches up", rejoin: false, salt: 800, donID: uint32(555333)}, + {name: "oracle rejoining after its job is deleted catches up", rejoin: true, salt: 850, donID: uint32(555444)}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testIntegrationLLOV31BlobCatchup(t, tc.rejoin, tc.salt, tc.donID) + }) + } +} + +func testIntegrationLLOV31BlobCatchup(t *testing.T, rejoin bool, salt int64, donID uint32) { + const ( + streamID = uint32(790) + channelID = uint32(1) + ) + // laggingNode is the index of the oracle that misses rounds. + laggingNode := nNodes - 1 + + offchainConfig := lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, + EnableObservationCompression: true, + } + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + for i := range nNodes { + k := big.NewInt(salt + int64(i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress, _, _, _, _ := setupBlockchain(t) + fromBlock := 1 + + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo_catchup", backend, bootstrapCSAKey, nil) + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + packetCh := make(chan *packet, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, serverKey, packetCh) + serverURL := startMercuryServer(t, srv, clientPubKeys) + + oracles, nodes := setupNodes(t, nNodes, backend, clientCSAKeys, func(c *chainlink.Config) { + c.Mercury.Transmitter.Protocol = new(mercurytransmitter.MercuryTransmitterProtocolGRPC) + }) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, fmt.Sprintf("job-catchup-%d", donID), relayType, relayConfig) + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d +pluginVersion = "v31"`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + + // One counter per node so the lagging oracle's observations can be told + // apart from the rest of the DON's. + bridgeCalls := make([]*atomic.Uint64, nNodes) + price := decimal.NewFromFloat(123.4) + addLaggingLLOJob := func() int32 { + return addLLOJob( + t, + nodes[laggingNode], + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[laggingNode], + "blob-catchup-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + var laggingJobID int32 + for i, node := range nodes { + bridgeCalls[i] = new(atomic.Uint64) + bridge := createSingleDecimalCountingBridge(t, "catchup", i, price, node.App.BridgeORM(), bridgeCalls[i]) + addSingleDecimalStreamJob(t, node, streamID, bridge) + if i == laggingNode { + // The late joiner gets no LLO job yet; the rejoining oracle gets one + // that is deleted below. + if rejoin { + laggingJobID = addLaggingLLOJob() + } + continue + } + addLLOJob( + t, + node, + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[i], + "blob-catchup-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + channelDefinitions := llotypes.ChannelDefinitions{ + channelID: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + {StreamID: streamID, Aggregator: llotypes.AggregatorMedian}, + }, + }, + } + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, + WithOracles(oracles), WithOffchainConfig(offchainConfig), WithV31(), + ) + + // reportsFlowing drains transmitted reports for the channel under test until + // it sees one or the deadline passes. + reportsFlowing := func(d time.Duration) bool { + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + pckt, errReceive := receiveWithTimeout(t, packetCh, time.Until(deadline)) + if errReceive != nil { + return false + } + req := pckt.req + if req.ReportFormat != uint32(llotypes.ReportFormatJSON) { + continue + } + _, _, r, _, errDecode := (lloreportcodec.JSONReportCodec{}).UnpackDecode(req.Payload) + if errDecode == nil && r.ChannelID == channelID { + return true + } + } + return false + } + + require.Eventually(t, func() bool { + return reportsFlowing(2 * time.Second) + }, reportTimeout, 100*time.Millisecond, "DON should report before the lagging oracle joins") + + if rejoin { + nodes[laggingNode].DeleteJob(t, laggingJobID) + // Wait for the deleted oracle to stop observing, so the rounds that + // follow genuinely advance without it. + var last uint64 + var stableSince time.Time + require.Eventually(t, func() bool { + c := bridgeCalls[laggingNode].Load() + if c != last { + last = c + stableSince = time.Now() + return false + } + return time.Since(stableSince) > 2*time.Second + }, 30*time.Second, 100*time.Millisecond, "deleted oracle should stop observing") + // Let the remaining oracles advance a good number of rounds alone. + require.True(t, reportsFlowing(10*time.Second), "DON should keep reporting without the deleted oracle") + } + + callsBeforeCatchup := bridgeCalls[laggingNode].Load() + if !rejoin { + require.Zero(t, callsBeforeCatchup, "oracle without an LLO job should not observe") + } + + laggingJobID = addLaggingLLOJob() + require.NotZero(t, laggingJobID) + + require.Eventually(t, func() bool { + return bridgeCalls[laggingNode].Load() > callsBeforeCatchup + }, reportTimeout, 100*time.Millisecond, "the lagging oracle should observe once it has caught up") + + require.True(t, reportsFlowing(10*time.Second), "DON should keep reporting after the lagging oracle catches up") +} + +// TestIntegration_LLO_v31_restart_resumes restarts one oracle of a reporting +// v31 DON. v31 keeps its protocol state in a pebble key-value store on disk +// rather than in an outcome carried between rounds, so a restarted oracle has +// to reopen that store, recover its LLO job from the database, and rejoin a +// protocol that advanced without it. +// +// The remaining 3 oracles (2F+1) keep the DON reporting throughout, so a +// restart that failed to recover would show up as an oracle that never resumes +// observing rather than as a stalled DON. +func TestIntegration_LLO_v31_restart_resumes(t *testing.T) { + t.Parallel() + + const ( + salt = int64(900) + donID = uint32(555555) + streamID = uint32(890) + channelID = uint32(1) + ) + // restartedNode is the index of the oracle that is stopped and started again. + restartedNode := nNodes - 1 + + offchainConfig := lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, + EnableObservationCompression: true, + } + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + for i := range nNodes { + k := big.NewInt(salt + int64(i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress, _, _, _, _ := setupBlockchain(t) + fromBlock := 1 + + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo_restart", backend, bootstrapCSAKey, nil) + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + packetCh := make(chan *packet, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, serverKey, packetCh) + serverURL := startMercuryServer(t, srv, clientPubKeys) + + oracles, nodes, restarts := setupNodesWithRestart(t, nNodes, backend, clientCSAKeys, func(c *chainlink.Config) { + c.Mercury.Transmitter.Protocol = new(mercurytransmitter.MercuryTransmitterProtocolGRPC) + }) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, "job-restart", relayType, relayConfig) + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d +pluginVersion = "v31"`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + + // One counter per node, so the restarted oracle's observations can be told + // apart from the rest of the DON's. + bridgeCalls := make([]*atomic.Uint64, nNodes) + price := decimal.NewFromFloat(456.7) + for i, node := range nodes { + bridgeCalls[i] = new(atomic.Uint64) + bridge := createSingleDecimalCountingBridge(t, "restart", i, price, node.App.BridgeORM(), bridgeCalls[i]) + addSingleDecimalStreamJob(t, node, streamID, bridge) + addLLOJob( + t, + node, + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[i], + "restart-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + channelDefinitions := llotypes.ChannelDefinitions{ + channelID: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + {StreamID: streamID, Aggregator: llotypes.AggregatorMedian}, + }, + }, + } + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, + WithOracles(oracles), WithOffchainConfig(offchainConfig), WithV31(), + ) + + reportsFlowing := func(d time.Duration) bool { + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + pckt, errReceive := receiveWithTimeout(t, packetCh, time.Until(deadline)) + if errReceive != nil { + return false + } + req := pckt.req + if req.ReportFormat != uint32(llotypes.ReportFormatJSON) { + continue + } + _, _, r, _, errDecode := (lloreportcodec.JSONReportCodec{}).UnpackDecode(req.Payload) + if errDecode == nil && r.ChannelID == channelID { + return true + } + } + return false + } + + require.Eventually(t, func() bool { + return reportsFlowing(2 * time.Second) + }, reportTimeout, 100*time.Millisecond, "DON should report before the restart") + require.Positive(t, bridgeCalls[restartedNode].Load(), "the oracle to restart should be observing first") + + // Restart. The new application reopens the same database and key-value + // store root, so nothing below re-creates the jobs. + var callsWhileDown uint64 + app := restarts[restartedNode](t, func() { + // While the oracle is down it must not observe, which is what makes the + // resumption below evidence of a real restart. + settled := bridgeCalls[restartedNode].Load() + require.Eventually(t, func() bool { + c := bridgeCalls[restartedNode].Load() + if c != settled { + settled = c + return false + } + return true + }, 15*time.Second, 500*time.Millisecond, "a stopped oracle should stop observing") + callsWhileDown = settled + require.True(t, reportsFlowing(10*time.Second), "DON should keep reporting while the oracle is down") + require.Equal(t, callsWhileDown, bridgeCalls[restartedNode].Load(), "a stopped oracle should not observe") + }) + nodes[restartedNode].App = app + + callsAfterRestart := bridgeCalls[restartedNode].Load() + require.Equal(t, callsWhileDown, callsAfterRestart, "no observations should have happened while the oracle was down") + require.Eventually(t, func() bool { + return bridgeCalls[restartedNode].Load() > callsAfterRestart + }, reportTimeout, 100*time.Millisecond, "the restarted oracle should resume observing without its jobs being re-created") + + require.True(t, reportsFlowing(10*time.Second), "DON should keep reporting after the restarted oracle rejoins") +} diff --git a/deployment/go.mod b/deployment/go.mod index a5c18580870..1f743e39fc0 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -48,7 +48,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d - github.com/smartcontractkit/chainlink-data-streams v1.1.1 + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 diff --git a/deployment/go.sum b/deployment/go.sum index 524ce8bf8e5..696a28f2166 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1409,8 +1409,8 @@ github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= diff --git a/go.mod b/go.mod index 73c4825a890..946e7aae56f 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 - github.com/smartcontractkit/chainlink-data-streams v1.1.1 + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260710181111-6417709a55ee diff --git a/go.sum b/go.sum index e067dafc04d..f258908d089 100644 --- a/go.sum +++ b/go.sum @@ -1110,8 +1110,8 @@ github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index f07c1d098d7..1315fb71ccb 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -410,7 +410,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 8b7eceeeb22..cd4fa742768 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1398,8 +1398,8 @@ github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 6f29b5d19f7..52186f75bce 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -481,7 +481,7 @@ require ( github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 0848f18e14f..693a7d426a1 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1636,8 +1636,8 @@ github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= diff --git a/plugins/plugins.public.yaml b/plugins/plugins.public.yaml index 2a2fdbaa244..e1dd66f2429 100644 --- a/plugins/plugins.public.yaml +++ b/plugins/plugins.public.yaml @@ -43,7 +43,7 @@ plugins: streams: - moduleURI: "github.com/smartcontractkit/chainlink-data-streams" - gitRef: "v1.1.1" + gitRef: "v1.1.2-0.20260929142841-14ba41fe8d23" installPath: "./mercury/cmd/chainlink-mercury" ton: diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 86b87f7c8eb..f1a412b4137 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -492,7 +492,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 381c70b04cc..893a2e27e16 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1662,8 +1662,8 @@ github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.202609151 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index ff3f84d756a..5e260a91d1d 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -624,7 +624,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index e44e29a7837..ee2dbf59075 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1843,8 +1843,8 @@ github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61 github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d h1:CZ0Om7lANyhpFJcmZE8RwQpEP4PyvoNLhOJckjm+zao= github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d/go.mod h1:Q5q/ohoAF5N7GXZS6QxgPWSaSLFtS6/53QHJbBUkCmU= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= From 3553a2746bba15bebb72a9d157a86050b4e73256 Mon Sep 17 00:00:00 2001 From: Bolek <1416262+bolekk@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:37:25 +0000 Subject: [PATCH 3/8] [CRE] Prune stale remote capability shims in the launcher (#23832) * Support refreshing CombinedClient info and idempotent shim Close() CombinedClient's embedded CapabilityInfo (including DON metadata) was frozen at construction with no way to update it, so DON member/family changes never propagated once a CombinedClient was cached. Add SetInfo to refresh it in place. triggerSubscriber/triggerPublisher.Close() closed a channel directly with no guard, panicking on a second call. Guard with sync.Once so future callers (e.g. pruning stale shims) can safely close a shim that might already be closed elsewhere. Prune stale remote capability shims in the launcher addRemoteCapabilityV2/serveCapabilityV2 only ever created or updated cachedShims entries - nothing ever removed one. When a capability, method, or DON dropped out of the registry (or a capability moved to a different DON ID), its trigger subscriber/publisher, executable client/server, and CombinedClient shims lingered forever: still registered with the dispatcher and, for CombinedClients, still occupying their capability ID in the local capabilities registry. Add computeWantedShimKeys, a dry-run pass over the same DON/capability config that mirrors addRemoteCapabilities/serveCapabilities' filtering without creating anything, and pruneStaleShims, which tears down any cached shim not in that wanted set: deregister from the dispatcher, Close() the shim, drop it from subServices, and (for CombinedClients) remove it from the capabilities registry. Pruning must run before the create/update loops, not after: a capability moving to a new DON ID needs its old CombinedClient removed from the registry first, since CombinedClient doesn't implement GetState() (registry.Add treats it as an unknown/unsafe-to-replace connectivity state and errors with ErrCapabilityAlreadyExists otherwise) and each DON ID gets a distinct CombinedClient object. Also switch shimKey from a formatted string to a small comparable struct, so pruning doesn't need to parse it back into capID/donID/ method. Add tests for shim pruning and capability DON reassignment Covers: a remote capability removed entirely from a DON's config (client-side shims and CombinedClient pruned, dispatcher deregistered); a served capability's method removed from a DON's config (server-side shims pruned); a capability moving to a different DON ID (old CombinedClient removed from the registry before the new one is added, without ErrCapabilityAlreadyExists); and CombinedClient.Info() reflecting updated DON metadata when the DON ID itself is unchanged. * Address review feedback: rename shimKey to shimID, use slices.Delete - Rename shimKey -> shimID per review: "key" was redundant since it's used as a general shim identifier, not just a map key. - Use slices.Delete instead of manual append-based slice removal in closeStaleShim. - Document that triggerSubscriber/triggerPublisher.Close() are safe to call more than once, with only the first call able to return an error; addresses review question about double-close swallowing the error. - Add changeset. SHARED-1014 --- core/capabilities/launcher.go | 207 +++++++++++++-- core/capabilities/launcher_test.go | 244 ++++++++++++++++++ core/capabilities/remote/combined_client.go | 9 + core/capabilities/remote/trigger_publisher.go | 30 ++- .../capabilities/remote/trigger_subscriber.go | 9 +- 5 files changed, 458 insertions(+), 41 deletions(-) diff --git a/core/capabilities/launcher.go b/core/capabilities/launcher.go index 24bf8c5adb7..0e99a1912f8 100644 --- a/core/capabilities/launcher.go +++ b/core/capabilities/launcher.go @@ -75,15 +75,19 @@ type launcher struct { // For V2 capabilities, shims are created once and their config is updated dynamically. type cachedShims struct { - combinedClients map[string]remote.CombinedClient - triggerSubscribers map[string]remote.TriggerSubscriber - triggerPublishers map[string]remote.TriggerPublisher - executableClients map[string]executable.Client - executableServers map[string]executable.Server + combinedClients map[shimID]remote.CombinedClient + triggerSubscribers map[shimID]remote.TriggerSubscriber + triggerPublishers map[shimID]remote.TriggerPublisher + executableClients map[shimID]executable.Client + executableServers map[shimID]executable.Server } -func shimKey(capID string, donID uint32, method string) string { - return fmt.Sprintf("%s:%d:%s", capID, donID, method) +// shimID identifies a single per-method remote shim. combinedClients are keyed with method == "", +// since a CombinedClient covers every method of a capability. +type shimID struct { + capID string + donID uint32 + method string } // TODO: add metric handler and instrument all the internal log.Error calls @@ -134,11 +138,11 @@ func NewLauncher( lggr: logger.Sugared(lggr).Named("CapabilitiesLauncher"), dispatcher: dispatcher, cachedShims: cachedShims{ - combinedClients: make(map[string]remote.CombinedClient), - triggerSubscribers: make(map[string]remote.TriggerSubscriber), - triggerPublishers: make(map[string]remote.TriggerPublisher), - executableClients: make(map[string]executable.Client), - executableServers: make(map[string]executable.Server), + combinedClients: make(map[shimID]remote.CombinedClient), + triggerSubscribers: make(map[shimID]remote.TriggerSubscriber), + triggerPublishers: make(map[shimID]remote.TriggerPublisher), + executableClients: make(map[shimID]executable.Client), + executableServers: make(map[shimID]executable.Server), }, registry: registry, workflowDonNotifier: workflowDonNotifier, @@ -345,6 +349,22 @@ func (w *launcher) onNewRegistry(ctx context.Context, metadataRegistry *registry w.lggr.Warn("My node doesn't belong to any DON families. No filtering will be applied.") } + belongsToAWorkflowDON := len(myWorkflowDONs) > 0 + belongsToACapabilityDON := len(myCapabilityDONs) > 0 + + // Prune shims for capabilities/methods/DONs that are no longer wanted, before the create/update + // loops below run. This must happen first: if a capability moved to a different DON ID, its old + // CombinedClient must be removed from the registry before the replacement is added under the new + // DON ID (registry.Add errors if an entry for the same capability ID is still present). + var wantedClientDONs, wantedServerDONs []registry.DON + if belongsToAWorkflowDON { + wantedClientDONs = remoteCapabilityDONs + } + if belongsToACapabilityDON { + wantedServerDONs = myCapabilityDONs + } + w.pruneStaleShims(ctx, computeWantedShimKeys(wantedClientDONs, wantedServerDONs)) + // Reconcile local capabilities: start/stop/restart capabilities based on registry state. if w.localCapMgr != nil { myDONs := make([]registry.DON, 0, len(myCapabilityDONs)+len(myWorkflowDONs)) @@ -355,7 +375,6 @@ func (w *launcher) onNewRegistry(ctx context.Context, metadataRegistry *registry } } - belongsToAWorkflowDON := len(myWorkflowDONs) > 0 if belongsToAWorkflowDON { myDON := myWorkflowDONs[0] @@ -373,7 +392,6 @@ func (w *launcher) onNewRegistry(ctx context.Context, metadataRegistry *registry } } - belongsToACapabilityDON := len(myCapabilityDONs) > 0 if belongsToACapabilityDON { for _, myDON := range myCapabilityDONs { w.serveCapabilities(ctx, w.myPeerID, myDON, remoteWorkflowDONs) @@ -547,9 +565,9 @@ func (w *launcher) addRemoteCapabilityV2(ctx context.Context, capID string, meth continue } - shimKey := shimKey(capID, remoteDON.ID, method) + key := shimID{capID: capID, donID: remoteDON.ID, method: method} if config.RemoteTriggerConfig != nil { // trigger - sub, alreadyExists := w.cachedShims.triggerSubscribers[shimKey] + sub, alreadyExists := w.cachedShims.triggerSubscribers[key] if !alreadyExists { sub = remote.NewTriggerSubscriber(capID, method, w.dispatcher, w.lggr) cc.SetTriggerSubscriber(method, sub) @@ -567,11 +585,11 @@ func (w *launcher) addRemoteCapabilityV2(ctx context.Context, capID string, meth w.lggr.Errorw("failed to start receiver", "capID", capID, "method", method, "error", err2) continue } - w.cachedShims.triggerSubscribers[shimKey] = sub + w.cachedShims.triggerSubscribers[key] = sub w.lggr.Infow("added new remote trigger subscriber", "capID", capID, "method", method) } } else { // executable - client, alreadyExists := w.cachedShims.executableClients[shimKey] + client, alreadyExists := w.cachedShims.executableClients[key] if !alreadyExists { client = executable.NewClient(info.ID, method, w.dispatcher, w.lggr) cc.SetExecutableClient(method, client) @@ -594,7 +612,7 @@ func (w *launcher) addRemoteCapabilityV2(ctx context.Context, capID string, meth w.lggr.Errorw("failed to start receiver", "capID", capID, "method", method, "error", err2) continue } - w.cachedShims.executableClients[shimKey] = client + w.cachedShims.executableClients[key] = client w.lggr.Infow("added new remote executable client", "capID", capID, "method", method) } } @@ -644,8 +662,8 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo if !ok { return fmt.Errorf("capability %s does not implement TriggerCapability", capID) } - shimKey := shimKey(capID, myDON.ID, method) - publisher, alreadyExists := w.cachedShims.triggerPublishers[shimKey] + key := shimID{capID: capID, donID: myDON.ID, method: method} + publisher, alreadyExists := w.cachedShims.triggerPublishers[key] if !alreadyExists { publisher = remote.NewTriggerPublisher( capID, @@ -665,7 +683,7 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo w.lggr.Errorw("failed to start receiver", "capID", capID, "method", method, "error", err2) continue } - w.cachedShims.triggerPublishers[shimKey] = publisher + w.cachedShims.triggerPublishers[key] = publisher w.lggr.Infow("added new remote trigger publisher", "capID", capID, "method", method) } } else { // executable @@ -674,8 +692,8 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo return fmt.Errorf("capability %s does not implement ExecutableCapability", capID) } - shimKey := shimKey(capID, myDON.ID, method) - server, alreadyExists := w.cachedShims.executableServers[shimKey] + key := shimID{capID: capID, donID: myDON.ID, method: method} + server, alreadyExists := w.cachedShims.executableServers[key] if !alreadyExists { server = executable.NewServer( info.ID, @@ -717,7 +735,7 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo w.lggr.Errorw("failed to start receiver", "capID", capID, "method", method, "error", err2) continue } - w.cachedShims.executableServers[shimKey] = server + w.cachedShims.executableServers[key] = server w.lggr.Infow("added new remote executable server", "capID", capID, "method", method) } } @@ -727,12 +745,149 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo // retrieve or create a CombinedClient for the given capability func (w *launcher) getCombinedClient(info capabilities.CapabilityInfo) (remote.CombinedClient, bool) { - key := shimKey(info.ID, info.DON.ID, "") // empty method name - CombinedClient covers all methods + key := shimID{capID: info.ID, donID: info.DON.ID} // empty method name - CombinedClient covers all methods cc, exists := w.cachedShims.combinedClients[key] if !exists { // create a new combined client and cache it cc = remote.NewCombinedClient(info) w.cachedShims.combinedClients[key] = cc return cc, true } + // DON ID is unchanged (same key), but other DON metadata (members, families, name, IsPublic) + // may have changed - refresh it so the CombinedClient's Info() doesn't go stale. + cc.SetInfo(info) return cc, false } + +// wantedShimKeys is the set of shim keys expected to exist after a reconcile pass. +type wantedShimKeys struct { + combinedClients map[shimID]struct{} + triggerSubscribers map[shimID]struct{} + executableClients map[shimID]struct{} + triggerPublishers map[shimID]struct{} + executableServers map[shimID]struct{} +} + +// computeWantedShimKeys determines every shim key this reconcile pass will create or refresh, so +// pruneStaleShims can remove stale cache entries before the create/update loops in onNewRegistry +// run. It mirrors the filtering addRemoteCapabilities/serveCapabilities apply (unmarshal errors, +// LocalOnly, missing method config) but only collects keys - it never creates or mutates anything. +// Pass nil for remoteCapabilityDONs/myCapabilityDONs when the node doesn't currently belong to a +// workflow/capability DON, matching the gating in onNewRegistry. +func computeWantedShimKeys(remoteCapabilityDONs, myCapabilityDONs []registry.DON) wantedShimKeys { + wanted := wantedShimKeys{ + combinedClients: map[shimID]struct{}{}, + triggerSubscribers: map[shimID]struct{}{}, + executableClients: map[shimID]struct{}{}, + triggerPublishers: map[shimID]struct{}{}, + executableServers: map[shimID]struct{}{}, + } + for _, remoteDON := range remoteCapabilityDONs { + for cid, c := range remoteDON.CapabilityConfigurations { + methodConfig, ok := unmarshalWantedMethodConfig(c) + if !ok { + continue + } + wanted.combinedClients[shimID{capID: cid, donID: remoteDON.ID}] = struct{}{} + for method, cfg := range methodConfig { + key := shimID{capID: cid, donID: remoteDON.ID, method: method} + switch { + case cfg.RemoteTriggerConfig != nil: + wanted.triggerSubscribers[key] = struct{}{} + case cfg.RemoteExecutableConfig != nil: + wanted.executableClients[key] = struct{}{} + } + } + } + } + for _, myDON := range myCapabilityDONs { + for cid, c := range myDON.CapabilityConfigurations { + methodConfig, ok := unmarshalWantedMethodConfig(c) + if !ok { + continue + } + for method, cfg := range methodConfig { + key := shimID{capID: cid, donID: myDON.ID, method: method} + switch { + case cfg.RemoteTriggerConfig != nil: + wanted.triggerPublishers[key] = struct{}{} + case cfg.RemoteExecutableConfig != nil: + wanted.executableServers[key] = struct{}{} + } + } + } + } + return wanted +} + +func unmarshalWantedMethodConfig(c registry.CapabilityConfiguration) (map[string]capabilities.CapabilityMethodConfig, bool) { + capabilityConfig, err := c.Unmarshal() + if err != nil || capabilityConfig.LocalOnly || capabilityConfig.CapabilityMethodConfig == nil { + return nil, false + } + return capabilityConfig.CapabilityMethodConfig, true +} + +// pruneStaleShims tears down cached shims that are no longer in the wanted set: capabilities or +// methods removed from a DON's config, DONs that disappeared from the registry, or capabilities +// that moved to a different DON ID. It must run before the create/update loops in onNewRegistry - +// see computeWantedShimKeys. +func (w *launcher) pruneStaleShims(ctx context.Context, wanted wantedShimKeys) { + for key := range w.cachedShims.combinedClients { + if _, ok := wanted.combinedClients[key]; ok { + continue + } + if err := w.registry.Remove(ctx, key.capID); err != nil { + w.lggr.Errorw("failed to remove stale combined client from registry", "capabilityID", key.capID, "donID", key.donID, "error", err) + } + delete(w.cachedShims.combinedClients, key) + } + + for key, sub := range w.cachedShims.triggerSubscribers { + if _, ok := wanted.triggerSubscribers[key]; ok { + continue + } + w.closeStaleShim(key, sub) + delete(w.cachedShims.triggerSubscribers, key) + } + for key, client := range w.cachedShims.executableClients { + if _, ok := wanted.executableClients[key]; ok { + continue + } + w.closeStaleShim(key, client) + delete(w.cachedShims.executableClients, key) + } + for key, pub := range w.cachedShims.triggerPublishers { + if _, ok := wanted.triggerPublishers[key]; ok { + continue + } + w.closeStaleShim(key, pub) + delete(w.cachedShims.triggerPublishers, key) + } + for key, server := range w.cachedShims.executableServers { + if _, ok := wanted.executableServers[key]; ok { + continue + } + w.closeStaleShim(key, server) + delete(w.cachedShims.executableServers, key) + } +} + +// closeStaleShim deregisters a pruned shim from the dispatcher and closes it, removing it from +// w.subServices so the launcher's own Close() doesn't close it a second time. +func (w *launcher) closeStaleShim(key shimID, shim any) { + w.dispatcher.RemoveReceiverForMethod(key.capID, key.donID, key.method) + if c, ok := shim.(interface{ Close() error }); ok { + if err := c.Close(); err != nil { + w.lggr.Errorw("failed to close stale shim", "capabilityID", key.capID, "donID", key.donID, "method", key.method, "error", err) + } + } + + w.muSubServices.Lock() + defer w.muSubServices.Unlock() + for i, s := range w.subServices { + if any(s) == shim { + w.subServices = slices.Delete(w.subServices, i, i+1) + break + } + } +} diff --git a/core/capabilities/launcher_test.go b/core/capabilities/launcher_test.go index 426c2397b05..d7f91800281 100644 --- a/core/capabilities/launcher_test.go +++ b/core/capabilities/launcher_test.go @@ -638,6 +638,250 @@ func TestLauncher_V2CapabilitiesAddViaCombinedClient(t *testing.T) { require.NoError(t, err) assert.Equal(t, fullTriggerCapID, capInfo.ID) assert.Len(t, capInfo.DON.Members, 5) + + // The CombinedClient's own Info() (not just the per-method subscriber's) must also reflect + // the updated DON metadata, even though the DON ID - and therefore the cache key - is unchanged. + ccInfo, err := trigCC.Info(t.Context()) + require.NoError(t, err) + assert.Len(t, ccInfo.DON.Members, 5) +} + +func TestLauncher_OnNewRegistry_PrunesRemovedRemoteCapability(t *testing.T) { + t.Parallel() + lggr := logger.Test(t) + reg := regpkg.NewRegistry(lggr) + dispatcher := remoteMocks.NewDispatcher(t) + + workflowDonNodes, capabilityDonNodes := newNodes(4), newNodes(4) + fullTriggerCapID := "streams-trigger@1.0.0" + triggerCapIDHash := RandomUTF8BytesWord() + wfDonID := uint32(1) + capDonID := uint32(2) + + triggerCfg, err := proto.Marshal(&capabilitiespb.CapabilityConfig{ + MethodConfigs: map[string]*capabilitiespb.CapabilityMethodConfig{ + "StreamsTrigger": { + RemoteConfig: &capabilitiespb.CapabilityMethodConfig_RemoteTriggerConfig{ + RemoteTriggerConfig: &capabilitiespb.RemoteTriggerConfig{ + RegistrationRefresh: durationpb.New(1 * time.Second), + MinResponsesToAggregate: 3, + }, + }, + }, + }, + }) + require.NoError(t, err) + + localRegistry := buildLocalRegistry() + addDON(localRegistry, wfDonID, 0, 1, true, true, workflowDonNodes, []string{"zone-a"}, 1, nil) + addDON(localRegistry, capDonID, 0, 1, true, false, capabilityDonNodes, []string{"zone-a"}, 1, [][32]byte{triggerCapIDHash}) + addCapabilityToDON(localRegistry, capDonID, fullTriggerCapID, capabilities.CapabilityTypeTrigger, triggerCfg) + + sharedPeer := mocks.NewSharedPeer(t) + sharedPeer.On("ID").Return(workflowDonNodes[0]) + sharedPeer.On("IsBootstrap").Return(false) + sharedPeer.On("UpdateConnectionsByDONs", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + launcher, err := NewLauncher( + lggr, + sharedPeer, + nil, + dispatcher, + reg, + &mockDonNotifier{}, limits.Factory{}, + false, 0, + ) + require.NoError(t, err) + servicetest.Run(t, launcher) + + dispatcher.On("SetReceiverForMethod", fullTriggerCapID, capDonID, "StreamsTrigger", mock.AnythingOfType("*remote.triggerSubscriber")).Return(nil).Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + require.Len(t, launcher.cachedShims.triggerSubscribers, 1) + require.Len(t, launcher.cachedShims.combinedClients, 1) + + capObj, err := reg.Get(t.Context(), fullTriggerCapID) + require.NoError(t, err) + _, err = capObj.Info(t.Context()) + require.NoError(t, err) + + // The capability is removed entirely from the remote capability DON's config. + capDon := localRegistry.IDsToDONs[regpkg.DonID(capDonID)] + capDon.CapabilityConfigurations = map[string]regpkg.CapabilityConfiguration{} + localRegistry.IDsToDONs[regpkg.DonID(capDonID)] = capDon + delete(localRegistry.IDsToCapabilities, fullTriggerCapID) + + dispatcher.On("RemoveReceiverForMethod", fullTriggerCapID, capDonID, "StreamsTrigger").Return().Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + + assert.Empty(t, launcher.cachedShims.triggerSubscribers) + assert.Empty(t, launcher.cachedShims.combinedClients) + assert.Empty(t, launcher.subServices) + + // registry.Remove doesn't delete the map entry, it nils out the wrapped capability - so Get + // still succeeds, but the capability itself is now unavailable. + capObj, err = reg.Get(t.Context(), fullTriggerCapID) + require.NoError(t, err) + _, err = capObj.Info(t.Context()) + require.Error(t, err) +} + +func TestLauncher_OnNewRegistry_HandlesCapabilityDONReassignment(t *testing.T) { + t.Parallel() + lggr := logger.Test(t) + reg := regpkg.NewRegistry(lggr) + dispatcher := remoteMocks.NewDispatcher(t) + + workflowDonNodes := newNodes(4) + capabilityDonNodesA, capabilityDonNodesB := newNodes(4), newNodes(4) + fullTriggerCapID := "streams-trigger@1.0.0" + triggerCapIDHash := RandomUTF8BytesWord() + wfDonID := uint32(1) + capDonIDA := uint32(2) + capDonIDB := uint32(3) + + triggerCfg, err := proto.Marshal(&capabilitiespb.CapabilityConfig{ + MethodConfigs: map[string]*capabilitiespb.CapabilityMethodConfig{ + "StreamsTrigger": { + RemoteConfig: &capabilitiespb.CapabilityMethodConfig_RemoteTriggerConfig{ + RemoteTriggerConfig: &capabilitiespb.RemoteTriggerConfig{ + RegistrationRefresh: durationpb.New(1 * time.Second), + MinResponsesToAggregate: 3, + }, + }, + }, + }, + }) + require.NoError(t, err) + + localRegistry := buildLocalRegistry() + addDON(localRegistry, wfDonID, 0, 1, true, true, workflowDonNodes, []string{"zone-a"}, 1, nil) + addDON(localRegistry, capDonIDA, 0, 1, true, false, capabilityDonNodesA, []string{"zone-a"}, 1, [][32]byte{triggerCapIDHash}) + addCapabilityToDON(localRegistry, capDonIDA, fullTriggerCapID, capabilities.CapabilityTypeTrigger, triggerCfg) + + sharedPeer := mocks.NewSharedPeer(t) + sharedPeer.On("ID").Return(workflowDonNodes[0]) + sharedPeer.On("IsBootstrap").Return(false) + sharedPeer.On("UpdateConnectionsByDONs", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + launcher, err := NewLauncher( + lggr, + sharedPeer, + nil, + dispatcher, + reg, + &mockDonNotifier{}, limits.Factory{}, + false, 0, + ) + require.NoError(t, err) + servicetest.Run(t, launcher) + + dispatcher.On("SetReceiverForMethod", fullTriggerCapID, capDonIDA, "StreamsTrigger", mock.AnythingOfType("*remote.triggerSubscriber")).Return(nil).Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + require.Len(t, launcher.cachedShims.combinedClients, 1) + + // The capability moves from capDonIDA to a brand new capDonIDB, disappearing from A's config + // entirely in the same registry snapshot. + capDonA := localRegistry.IDsToDONs[regpkg.DonID(capDonIDA)] + capDonA.CapabilityConfigurations = map[string]regpkg.CapabilityConfiguration{} + localRegistry.IDsToDONs[regpkg.DonID(capDonIDA)] = capDonA + + addDON(localRegistry, capDonIDB, 0, 1, true, false, capabilityDonNodesB, []string{"zone-a"}, 1, [][32]byte{triggerCapIDHash}) + addCapabilityToDON(localRegistry, capDonIDB, fullTriggerCapID, capabilities.CapabilityTypeTrigger, triggerCfg) + + dispatcher.On("RemoveReceiverForMethod", fullTriggerCapID, capDonIDA, "StreamsTrigger").Return().Once() + dispatcher.On("SetReceiverForMethod", fullTriggerCapID, capDonIDB, "StreamsTrigger", mock.AnythingOfType("*remote.triggerSubscriber")).Return(nil).Once() + + // Must not fail with ErrCapabilityAlreadyExists: the stale combinedClient under capDonIDA has + // to be pruned (and removed from the registry) before the new one under capDonIDB is added. + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + + require.Len(t, launcher.cachedShims.combinedClients, 1) + for key := range launcher.cachedShims.combinedClients { + assert.Equal(t, capDonIDB, key.donID) + } + + capObj, err := reg.Get(t.Context(), fullTriggerCapID) + require.NoError(t, err) + info, err := capObj.Info(t.Context()) + require.NoError(t, err) + assert.Equal(t, capDonIDB, info.DON.ID) +} + +func TestLauncher_OnNewRegistry_PrunesRemovedServedCapability(t *testing.T) { + t.Parallel() + lggr := logger.Test(t) + reg := regpkg.NewRegistry(lggr) + fullTriggerCapID := "streams-trigger@1.0.0" + mt := newMockTrigger(capabilities.MustNewCapabilityInfo( + fullTriggerCapID, + capabilities.CapabilityTypeTrigger, + "streams trigger", + )) + require.NoError(t, reg.Add(t.Context(), mt)) + + dispatcher := remoteMocks.NewDispatcher(t) + + workflowDonNodes, capabilityDonNodes := newNodes(4), newNodes(4) + triggerCapID := RandomUTF8BytesWord() + wfDonID := uint32(1) + capDonID := uint32(2) + + triggerCfg, err := proto.Marshal(&capabilitiespb.CapabilityConfig{ + MethodConfigs: map[string]*capabilitiespb.CapabilityMethodConfig{ + "StreamsTrigger": { + RemoteConfig: &capabilitiespb.CapabilityMethodConfig_RemoteTriggerConfig{ + RemoteTriggerConfig: &capabilitiespb.RemoteTriggerConfig{ + RegistrationRefresh: durationpb.New(1 * time.Second), + MinResponsesToAggregate: 3, + }, + }, + }, + }, + }) + require.NoError(t, err) + + localRegistry := buildLocalRegistry() + addDON(localRegistry, wfDonID, 0, 1, true, true, workflowDonNodes, []string{"zone-a"}, 1, nil) + addDON(localRegistry, capDonID, 0, 1, true, false, capabilityDonNodes, []string{"zone-a"}, 1, [][32]byte{triggerCapID}) + addCapabilityToDON(localRegistry, capDonID, fullTriggerCapID, capabilities.CapabilityTypeTrigger, triggerCfg) + + sharedPeer := mocks.NewSharedPeer(t) + sharedPeer.On("ID").Return(capabilityDonNodes[0]) + sharedPeer.On("IsBootstrap").Return(false) + sharedPeer.On("UpdateConnectionsByDONs", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + launcher, err := NewLauncher( + lggr, + sharedPeer, + nil, + dispatcher, + reg, + &mockDonNotifier{}, limits.Factory{}, + false, 0, + ) + require.NoError(t, err) + require.NoError(t, launcher.Start(t.Context())) + defer launcher.Close() + + dispatcher.On("SetReceiverForMethod", fullTriggerCapID, capDonID, "StreamsTrigger", mock.AnythingOfType("*remote.triggerPublisher")).Return(nil).Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + require.Len(t, launcher.cachedShims.triggerPublishers, 1) + + // The method is removed from the capability DON's config (e.g. the capability no longer hosts it). + capDon := localRegistry.IDsToDONs[regpkg.DonID(capDonID)] + capDon.CapabilityConfigurations = map[string]regpkg.CapabilityConfiguration{} + localRegistry.IDsToDONs[regpkg.DonID(capDonID)] = capDon + + dispatcher.On("RemoveReceiverForMethod", fullTriggerCapID, capDonID, "StreamsTrigger").Return().Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + + assert.Empty(t, launcher.cachedShims.triggerPublishers) + assert.Empty(t, launcher.subServices) } func TestLauncher_V2CapabilitiesExposeRemotely(t *testing.T) { diff --git a/core/capabilities/remote/combined_client.go b/core/capabilities/remote/combined_client.go index 74447a6a5f1..4a0d9ec09c0 100644 --- a/core/capabilities/remote/combined_client.go +++ b/core/capabilities/remote/combined_client.go @@ -20,6 +20,7 @@ type CombinedClient interface { SetExecutableClient(method string, client capabilities.ExecutableCapability) GetTriggerSubscriber(method string) capabilities.TriggerCapability GetExecutableClient(method string) capabilities.ExecutableCapability + SetInfo(info capabilities.CapabilityInfo) } type combinedClient struct { @@ -32,6 +33,8 @@ type combinedClient struct { var _ CombinedClient = &combinedClient{} func (c *combinedClient) Info(ctx context.Context) (capabilities.CapabilityInfo, error) { + c.mu.RLock() + defer c.mu.RUnlock() return c.info, nil } @@ -118,3 +121,9 @@ func (c *combinedClient) GetExecutableClient(method string) capabilities.Executa defer c.mu.RUnlock() return c.executableClients[method] } + +func (c *combinedClient) SetInfo(info capabilities.CapabilityInfo) { + c.mu.Lock() + defer c.mu.Unlock() + c.info = info +} diff --git a/core/capabilities/remote/trigger_publisher.go b/core/capabilities/remote/trigger_publisher.go index 2ae1b02989d..dd18098e4f7 100644 --- a/core/capabilities/remote/trigger_publisher.go +++ b/core/capabilities/remote/trigger_publisher.go @@ -48,6 +48,7 @@ type triggerPublisher struct { bqMu sync.Mutex // protects batchingQueue stopCh services.StopChan wg sync.WaitGroup + closeOnce sync.Once ackExecutor *ParallelExecutor registerExecutor *ParallelExecutor lggr logger.Logger @@ -929,22 +930,27 @@ func (p *triggerPublisher) batchingLoop() { } func (p *triggerPublisher) Close() error { - close(p.stopCh) + var err error + p.closeOnce.Do(func() { + close(p.stopCh) - if p.ackExecutor != nil { - if err := p.ackExecutor.Close(); err != nil { - return fmt.Errorf("failed to close ack executor: %w", err) + if p.ackExecutor != nil { + if e := p.ackExecutor.Close(); e != nil { + err = fmt.Errorf("failed to close ack executor: %w", e) + return + } } - } - if p.registerExecutor != nil { - if err := p.registerExecutor.Close(); err != nil { - return fmt.Errorf("failed to close register executor: %w", err) + if p.registerExecutor != nil { + if e := p.registerExecutor.Close(); e != nil { + err = fmt.Errorf("failed to close register executor: %w", e) + return + } } - } - p.wg.Wait() - p.lggr.Info("TriggerPublisher closed") - return nil + p.wg.Wait() + p.lggr.Info("TriggerPublisher closed") + }) + return err } func (p *triggerPublisher) Ready() error { diff --git a/core/capabilities/remote/trigger_subscriber.go b/core/capabilities/remote/trigger_subscriber.go index 3bcb0bbdc3e..192f4bda710 100644 --- a/core/capabilities/remote/trigger_subscriber.go +++ b/core/capabilities/remote/trigger_subscriber.go @@ -44,6 +44,7 @@ type triggerSubscriber struct { mu sync.RWMutex // protects registeredWorkflows, messageCache, and ackReplayCache stopCh services.StopChan wg sync.WaitGroup + closeOnce sync.Once lggr logger.Logger } @@ -495,9 +496,11 @@ func (s *triggerSubscriber) eventCleanupLoop() { } func (s *triggerSubscriber) Close() error { - close(s.stopCh) - s.wg.Wait() - s.lggr.Info("TriggerSubscriber closed") + s.closeOnce.Do(func() { + close(s.stopCh) + s.wg.Wait() + s.lggr.Info("TriggerSubscriber closed") + }) return nil } From 1b98ec94595ef0e9d0036a838c6e1cf5e9985a5a Mon Sep 17 00:00:00 2001 From: Gabriel Paradiso Date: Tue, 29 Sep 2026 17:02:11 +0000 Subject: [PATCH 4/8] [CRE-6745] wire noop coordinator + feature flag (#23837) * feat: add CoordinatedEngine featureflag * chore: clean up unused OnSubscriptionsReady * feat: define TriggerCoordinator interface * feat: wire the noop trigger coordinator * feat: add metrics to count type of engines * test: cleanup unused OnSubscriptionsReady * chore: refactor tryEngineCreate to have clear separate path for legacy and coordinated engine * fix: lint issues * chore: log errors, improve logs, explicit return on err * chore: bump chainlink-common and use cresettings.Default.CoordinatedEngineEnabled * fix: check engine is coordinated, as triggerCoordinator is always wired * fix: remove coordinated from the engine registry, type assert this from the engine itself * refactor: use one factory for legacy and coordinated engines * chore: build cleanup func to dedup * fix: lint issues * chore: newEngine calls useCoordinatedEngine instead of forcing the factoryfn to receive the parameter * test: add tests covering sharding + alternative scenarios for coordinated engine * chore: return ErrCoordinatedShardingUnsupported on sharding coordinated engine methods --- core/scripts/go.mod | 2 +- core/scripts/go.sum | 4 +- core/services/cre/cre.go | 3 + .../workflows/syncer/v2/engine_registry.go | 8 + .../syncer/v2/engine_registry_test.go | 108 ++++++ core/services/workflows/syncer/v2/handler.go | 331 ++++++++++++++---- .../syncer/v2/handler_engine_create_test.go | 276 +++++++++++++++ .../workflows/syncer/v2/handler_test.go | 25 +- core/services/workflows/syncer/v2/metrics.go | 40 ++- .../syncer/v2/shard_failover_manager.go | 34 +- .../syncer/v2/shard_failover_manager_test.go | 48 +++ .../syncer/v2/trigger_coordinator.go | 103 ++++++ .../workflows/syncer/v2/workflow_registry.go | 42 ++- .../syncer/v2/workflow_syncer_v2_test.go | 2 +- core/services/workflows/v2/config.go | 20 +- core/services/workflows/v2/engine_test.go | 4 - deployment/go.mod | 2 +- deployment/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- system-tests/lib/go.mod | 2 +- system-tests/lib/go.sum | 4 +- system-tests/tests/go.mod | 2 +- system-tests/tests/go.sum | 4 +- 28 files changed, 957 insertions(+), 129 deletions(-) create mode 100644 core/services/workflows/syncer/v2/handler_engine_create_test.go create mode 100644 core/services/workflows/syncer/v2/trigger_coordinator.go diff --git a/core/scripts/go.mod b/core/scripts/go.mod index ee94f14be06..a720949d3df 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -33,7 +33,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 526acfb1ff3..fa9116d6059 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1683,8 +1683,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 1e9b2046986..834cb5d07c2 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -973,6 +973,9 @@ func newWorkflowRegistrySyncerV2( syncerV2.WithShardRoutingSteady(shardRoutingSteady), syncerV2.WithShardResolver(shardResolver), syncerV2.WithShardIndex(uint32(cfg.Sharding().ShardIndex())), + syncerV2.WithTriggerCoordinator( + syncerV2.NewTriggerCoordinator(opts.CapabilitiesRegistry, engineRegistry, clockwork.NewRealClock(), lggr), + ), } if shardingEnabled && dispatcher != nil { handlerOpts = append(handlerOpts, diff --git a/core/services/workflows/syncer/v2/engine_registry.go b/core/services/workflows/syncer/v2/engine_registry.go index 82eba6d951d..f360a1f0132 100644 --- a/core/services/workflows/syncer/v2/engine_registry.go +++ b/core/services/workflows/syncer/v2/engine_registry.go @@ -9,6 +9,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/hashutil" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" ) var ErrNotFound = errors.New("engine not found") @@ -23,6 +24,13 @@ type ServiceWithMetadata struct { services.Service } +// Coordinated is true for an engine that leaves trigger registration, +// dequeuing and acknowledgement to the TriggerCoordinator. +func (e ServiceWithMetadata) Coordinated() bool { + engine, ok := e.Service.(v2.WorkflowEngine) + return ok && engine.IsCoordinated() +} + // engineEntry holds the engine and its associated source for internal storage. type engineEntry struct { engine services.Service diff --git a/core/services/workflows/syncer/v2/engine_registry_test.go b/core/services/workflows/syncer/v2/engine_registry_test.go index 09ca7e02850..92270c0edb6 100644 --- a/core/services/workflows/syncer/v2/engine_registry_test.go +++ b/core/services/workflows/syncer/v2/engine_registry_test.go @@ -3,11 +3,13 @@ package v2 import ( "context" "testing" + "time" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" ) func TestEngineRegistry(t *testing.T) { @@ -168,3 +170,109 @@ func (f fakeService) Ready() error { return nil } func (f fakeService) HealthReport() map[string]error { return map[string]error{} } func (f fakeService) Name() string { return "" } + +// fakeDrainableService reports a fixed drain state; it is otherwise a fakeService. +type fakeDrainableService struct { + fakeService + draining bool +} + +func (f *fakeDrainableService) Drain() bool { return false } + +func (f *fakeDrainableService) ActiveExecutions() int32 { return 0 } + +func (f *fakeDrainableService) DrainStartedAt() (time.Time, bool) { + if !f.draining { + return time.Time{}, false + } + return time.Unix(1, 0), true +} + +// fakeCoordinatedDrainableEngine is a v2.WorkflowEngine that reports as +// coordinated and drainable. Only the methods ServiceWithMetadata.Coordinated +// and the DrainableService assertion touch are implemented. +type fakeCoordinatedDrainableEngine struct { + v2.WorkflowEngine + draining bool +} + +func (f *fakeCoordinatedDrainableEngine) IsCoordinated() bool { return true } + +func (f *fakeCoordinatedDrainableEngine) Drain() bool { return false } + +func (f *fakeCoordinatedDrainableEngine) ActiveExecutions() int32 { return 0 } + +func (f *fakeCoordinatedDrainableEngine) DrainStartedAt() (time.Time, bool) { + if !f.draining { + return time.Time{}, false + } + return time.Unix(1, 0), true +} + +func TestEngineRegistry_Coordinated(t *testing.T) { + t.Parallel() + legacyID := types.WorkflowID([32]byte{1}) + coordinatedID := types.WorkflowID([32]byte{2}) + + er := NewEngineRegistry() + require.NoError(t, er.AddWithReconcileKey(legacyID, "TestSource", "legacy-key", &fakeService{})) + require.NoError(t, er.AddWithReconcileKey(coordinatedID, "TestSource", "coordinated-key", &fakeCoordinatedDrainableEngine{})) + require.ErrorIs(t, er.AddWithReconcileKey(coordinatedID, "TestSource", "coordinated-key", &fakeCoordinatedDrainableEngine{}), ErrAlreadyExists) + + entry, ok := er.Get(coordinatedID) + require.True(t, ok) + require.True(t, entry.Coordinated()) + require.Equal(t, "coordinated-key", entry.ReconcileKey) + + entry, ok = er.Get(legacyID) + require.True(t, ok) + require.False(t, entry.Coordinated()) + + byID := map[types.WorkflowID]bool{} + for _, e := range er.GetAll() { + byID[e.WorkflowID] = e.Coordinated() + } + require.Equal(t, map[types.WorkflowID]bool{legacyID: false, coordinatedID: true}, byID) + + for _, e := range er.GetBySource("TestSource") { + require.Equal(t, e.WorkflowID == coordinatedID, e.Coordinated()) + } + + popped, err := er.Pop(coordinatedID) + require.NoError(t, err) + require.True(t, popped.Coordinated()) + + for _, e := range er.PopAll() { + require.False(t, e.Coordinated()) + } +} + +func TestCountEngines(t *testing.T) { + t.Parallel() + + t.Run("empty registry", func(t *testing.T) { + t.Parallel() + require.Equal(t, engineCounts{}, countEngines(nil)) + }) + + t.Run("mixed engine types", func(t *testing.T) { + t.Parallel() + engines := []ServiceWithMetadata{ + {Service: &fakeCoordinatedDrainableEngine{}}, + {Service: &fakeCoordinatedDrainableEngine{draining: true}}, + {Service: &fakeDrainableService{}}, + {Service: &fakeDrainableService{draining: true}}, + {Service: &fakeService{}}, + {Service: newTestShardFailoverManager(t, &mockEngine{})}, + } + require.Equal(t, engineCounts{draining: 2, coordinated: 2, legacy: 4}, countEngines(engines)) + }) + + t.Run("a sharded wrapper follows the engine it wraps", func(t *testing.T) { + t.Parallel() + engines := []ServiceWithMetadata{ + {Service: newTestShardFailoverManager(t, &fakeCoordinatedDrainableEngine{})}, + } + require.Equal(t, engineCounts{coordinated: 1}, countEngines(engines)) + }) +} diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index f0ffcae60d5..cbb69bd4e6c 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -56,11 +56,9 @@ type ORM interface { artifacts.WorkflowSpecsDS } -// engineFactoryFn creates a workflow engine. The initDone channel is used to signal when the engine -// has completed initialization (including trigger subscriptions). For v2 engines, this is wired to -// the OnInitialized lifecycle hook. For v1 legacy DAG engines, nil is sent immediately after engine -// creation since they don't support async initialization hooks. -type engineFactoryFn func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) +// engineFactoryFn creates a workflow engine. The initDone channel is used to signal when +// the engine has completed initialization; it is wired to the OnInitialized lifecycle hook. +type engineFactoryFn func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) type DrainableService interface { Drain() bool @@ -77,19 +75,26 @@ type eventHandler struct { lggr logger.Logger - workflowStore store.Store - capRegistry registry.CapabilitiesRegistry - executionHandlers *confidentialrelay.ExecutionHandlers - donTimeStore *dontime.Store - useLocalTimeProvider bool - engineRegistry *EngineRegistry - emitter custmsg.MessageEmitter - emitterMu sync.RWMutex - engineFactory engineFactoryFn - engineLimiters *v2.EngineLimiters - featureFlags *v2.EngineFeatureFlags - ratelimiter *ratelimiter.RateLimiter - workflowLimits limits.ResourceLimiter[int] + workflowStore store.Store + capRegistry registry.CapabilitiesRegistry + executionHandlers *confidentialrelay.ExecutionHandlers + donTimeStore *dontime.Store + useLocalTimeProvider bool + engineRegistry *EngineRegistry + emitter custmsg.MessageEmitter + emitterMu sync.RWMutex + engineFactory engineFactoryFn + engineLimiters *v2.EngineLimiters + featureFlags *v2.EngineFeatureFlags + ratelimiter *ratelimiter.RateLimiter + workflowLimits limits.ResourceLimiter[int] + + // triggerCoordinator owns registration/handles/ACK for every workflow + // running the coordinated engine. Nil until wired via WithTriggerCoordinator. + // useCoordinatedEngine picks the legacy path when nil, regardless of the + // CoordinatedEngine flag. + triggerCoordinator TriggerCoordinator + workflowArtifactsStore WorkflowArtifactsStore workflowEncryptionKey workflowkey.Key workflowDonSubscriber capabilities.DonSubscriber @@ -152,9 +157,9 @@ func WithEngineFactoryFn(efn engineFactoryFn) func(*eventHandler) { } } -func WithStaticEngine(engine services.Service) func(*eventHandler) { +func WithStaticEngine(engine v2.WorkflowEngine) func(*eventHandler) { return func(e *eventHandler) { - e.engineFactory = func(_ context.Context, _, _ string, _ types.WorkflowName, _ string, _, _ []byte, _ string, initDone chan<- error) (services.Service, error) { + e.engineFactory = func(_ context.Context, _, _ string, _ types.WorkflowName, _ string, _, _ []byte, _ string, initDone chan<- error) (v2.WorkflowEngine, error) { // For static engines (used in tests), signal immediate initialization success if initDone != nil { initDone <- nil @@ -164,6 +169,16 @@ func WithStaticEngine(engine services.Service) func(*eventHandler) { } } +// WithTriggerCoordinator wires the TriggerCoordinator used for every workflow +// the CoordinatedEngine flag routes to the coordinated engine. Without +// this option, tryEngineCreate always takes the legacy path, regardless of +// the flag's value. +func WithTriggerCoordinator(tc TriggerCoordinator) func(*eventHandler) { + return func(e *eventHandler) { + e.triggerCoordinator = tc + } +} + func WithBillingClient(client metering.BillingClient) func(*eventHandler) { return func(e *eventHandler) { e.billingClient = client @@ -367,15 +382,15 @@ func NewEventHandler( return nil, fmt.Errorf("new metrics: %w", metricsErr) } eh.metrics = metricsInst - eh.engineFactory = eh.engineFactoryFn + eh.engineFactory = eh.newEngine for _, o := range opts { o(eh) } eh.Service, eh.eng = services.Config{ Name: "EventHandler", - // The workflow store and the spec meter are started and stopped - // alongside the handler. + // The workflow store, the spec meter and the trigger coordinator are + // started and stopped alongside the handler. NewSubServices: func(logger.Logger) []services.Service { var subs []services.Service if eh.workflowStore != nil { @@ -384,6 +399,9 @@ func NewEventHandler( if eh.specMeter != nil { subs = append(subs, eh.specMeter) } + if eh.triggerCoordinator != nil { + subs = append(subs, eh.triggerCoordinator) + } return subs }, Start: eh.start, @@ -855,9 +873,32 @@ func (h *eventHandler) fetchOrganizationID(ctx context.Context, workflowOwner st return organizationID, nil } -func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) { +// useCoordinatedEngine reads the CoordinatedEngine flag. A node without a +// TriggerCoordinator always takes the legacy path, and so does a sharded node: +// sharding and the coordinated engine are not yet compatible. +func (h *eventHandler) useCoordinatedEngine(ctx context.Context, workflowID string) bool { + if h.triggerCoordinator == nil || h.featureFlags == nil || h.featureFlags.CoordinatedEngine == nil { + return false + } + if h.shardingEnabled && h.dispatcher != nil { + return false + } + open, err := h.featureFlags.CoordinatedEngine.IsOpen(ctx) + if err != nil { + h.lggr.Warnw("Could not evaluate CoordinatedEngine flag, falling back to the legacy engine", + "workflowID", workflowID, "err", err) + return false + } + return open +} + +// buildEngineConfig builds the module stack (local WASM module plus the +// confidential module) and the EngineConfig that the legacy and the +// coordinated engine share. +func (h *eventHandler) buildEngineConfig(ctx context.Context, workflowID, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (*v2.EngineConfig, error) { lggr := logger.Named(h.lggr, "WorkflowEngine.Module") lggr = logger.With(lggr, "workflowID", workflowID, "workflowName", name, "workflowOwner", owner) + var sdkName string moduleConfig := &host.ModuleConfig{ Logger: lggr, @@ -888,7 +929,6 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID, owner st h.lggr.Debugw("Finished creating module for workflowID", "workflowID", workflowID) - // V2 aka "NoDAG" // Wrap the local WASM module in a RequirementSelectingModule that routes // triggers with a TEE requirement to the ConfidentialModule (which delegates // to the confidential-workflows capability and runs the WASM inside the @@ -912,6 +952,21 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID, owner st h.wireInitDoneHook(cfg, initDone) + return cfg, nil +} + +// newEngine is the default engineFactoryFn: it builds the module stack and +// EngineConfig, wires sharding, and calls one of the two engine constructors. +// +// The coordinated engine registers no triggers itself: +// tryCoordinatedEngineCreate hands it to the coordinator once it is in the +// registry, and the coordinator calls Subscribe to obtain its subscriptions. +func (h *eventHandler) newEngine(ctx context.Context, workflowID, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) { + cfg, err := h.buildEngineConfig(ctx, workflowID, owner, name, tag, config, binary, binaryURL, initDone) + if err != nil { + return nil, err + } + var manager *ShardFailoverManager if h.shardingEnabled && h.dispatcher != nil { if h.shardFailoverComm == nil { @@ -935,7 +990,18 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID, owner st manager.WireHooks(cfg) } - engine, err := v2.NewEngine(cfg) + construct := v2.NewEngine + if h.useCoordinatedEngine(ctx, workflowID) { + // The trigger coordinator is not implemented yet, so a workflow routed here + // will never receive triggers. Warn once per engine creation with the workflowID + h.lggr.Warnw("Routing workflow to the coordinated engine; trigger delivery is not implemented yet", + "workflowID", workflowID, "workflowName", name, "workflowOwner", owner) + + cfg.TriggerAcknowledger = h.triggerCoordinator + construct = v2.NewCoordinatedEngine + } + + engine, err := construct(cfg) if err != nil { return nil, err } @@ -981,6 +1047,14 @@ func (h *eventHandler) stopEngine(ctx context.Context, workflowID types.Workflow e, ok := h.engineRegistry.Get(workflowID) var drainable DrainableService if ok { + // on coordinated engines, stop coordinator ingress before draining, + // so the drain can actually reach zero active executions. + if e.Coordinated() && h.triggerCoordinator != nil { + if err := h.triggerCoordinator.UnregisterTriggers(workflowID.Hex()); err != nil && !errors.Is(err, ErrWorkflowNotCoordinated) { + h.lggr.Errorw("Failed to unregister triggers via coordinator", "workflowID", workflowID.String(), "err", err) + } + } + var isDrainable bool if drainable, isDrainable = e.Service.(DrainableService); isDrainable { if started := drainable.Drain(); started { @@ -1106,6 +1180,16 @@ func (h *eventHandler) tryEngineCleanup(workflowID types.WorkflowID) error { return nil } + // Unregister before close, same as stopEngine. + // This path (reconcile-to-inactive, replace-draining-engine) must mirror + // stopEngine's coordinator handling, or trigger registrations are left + // orphaned on this path alone. + if e.Coordinated() && h.triggerCoordinator != nil { + if err := h.triggerCoordinator.UnregisterTriggers(workflowID.Hex()); err != nil && !errors.Is(err, ErrWorkflowNotCoordinated) { + h.lggr.Errorw("Failed to unregister triggers via coordinator", "workflowID", workflowID.String(), "err", err) + } + } + // Close the engine, then remove it from the registry only once cleanup has succeeded. The // registry entry is what tells us this workflow still needs cleanup, so if a step fails we // leave it in place and return the error, allowing a future attempt to retry in case the @@ -1134,9 +1218,22 @@ func (h *eventHandler) cleanupModuleCache(workflowID string) { } } +// engineInputs is the validated, decoded form of a WorkflowSpec that both +// engine creation paths start from. +type engineInputs struct { + wid types.WorkflowID + workflowName types.WorkflowName + binary []byte + config []byte + reconcileKey string +} + // tryEngineCreate attempts to create a new workflow engine, start it, and register it with the engine registry. // This function waits for the engine to complete initialization (including trigger subscriptions) before returning, // ensuring that the workflowActivated event accurately reflects the deployment status including trigger registration. +// +// The CoordinatedEngine flag is read here, once, and decides between the legacy +// and the coordinated creation path. func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSpec, source string) error { ctx, span := h.tracer.Start(ctx, "engine_create", trace.WithAttributes( @@ -1146,15 +1243,102 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp )) defer span.End() + in, err := h.prepareEngineInputs(ctx, spec) + if err != nil { + return err + } + + initDone := make(chan error, 1) + engine, err := h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, in.workflowName, spec.WorkflowTag, in.config, in.binary, spec.BinaryURL, initDone) + if err != nil { + return fmt.Errorf("failed to create workflow engine: %w", err) + } + + if engine.IsCoordinated() { + return h.tryCoordinatedEngineCreate(ctx, spec, source, in, engine, initDone) + } + return h.tryLegacyEngineCreate(ctx, spec, source, in, engine, initDone) +} + +// tryLegacyEngineCreate finishes creation for an engine that registers its own +// triggers while it starts, so it is done once the engine is in the registry. +func (h *eventHandler) tryLegacyEngineCreate(ctx context.Context, spec *job.WorkflowSpec, source string, in *engineInputs, engine v2.WorkflowEngine, initDone <-chan error) error { + if err := h.startEngine(ctx, spec.WorkflowID, engine, initDone); err != nil { + return err + } + + if err := h.engineRegistry.AddWithReconcileKey(in.wid, source, in.reconcileKey, engine); err != nil { + return h.registryAddFailed(in.wid, source, engine, err) + } + return nil +} + +// tryCoordinatedEngineCreate finishes creation for an engine that registers no triggers itself. +func (h *eventHandler) tryCoordinatedEngineCreate(ctx context.Context, spec *job.WorkflowSpec, source string, in *engineInputs, engine v2.WorkflowEngine, initDone <-chan error) error { + if err := h.startEngine(ctx, spec.WorkflowID, engine, initDone); err != nil { + return err + } + + if err := h.engineRegistry.AddWithReconcileKey(in.wid, source, in.reconcileKey, engine); err != nil { + return h.registryAddFailed(in.wid, source, engine, err) + } + + cleanup := func(location string) { + if _, innerErr := h.engineRegistry.Pop(in.wid); innerErr != nil { + h.lggr.Warnw(fmt.Sprintf("Failed to pop engine from registry after %s failure", location), "workflowID", in.wid.Hex(), "err", innerErr) + } + if innerErr := engine.Close(); innerErr != nil { + h.lggr.Warnw(fmt.Sprintf("Failed to close engine after %s failure", location), "workflowID", in.wid.Hex(), "err", innerErr) + } + } + + // Registration happens here, now that the engine is already in the + // registry, so the coordinator's readers can resolve it for the first + // event. RegisterTriggers calls the engine's Subscribe itself — the WASM + // call is scoped to the one workflow it describes, which matters because + // Handle runs on a worker pool and several workflows reach this point + // concurrently. + localNode, err := h.capRegistry.LocalNode(ctx) + if err != nil { + cleanup("local node resolution") + return fmt.Errorf("failed to resolve local node DON ID for trigger registration: %w", err) + } + donID := localNode.WorkflowDON.ID + + triggerIDs, err := h.triggerCoordinator.RegisterTriggers(ctx, engine, RegistrationParams{ + WorkflowOwner: spec.WorkflowOwner, + WorkflowName: in.workflowName.Hex(), + DecodedWorkflowName: in.workflowName.String(), + WorkflowTag: spec.WorkflowTag, + WorkflowDonID: donID, + // pinnedWorkflowDonConfigVersion in v2 pins this to 1 to avoid forcing + // forwarder updates on config churn; mirrored here since the syncer + // can't reference that unexported v2 constant. + WorkflowDonConfigVersion: 1, + WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, + WorkflowRegistryAddress: h.workflowRegistryAddress, + }) + if err != nil { + cleanup("trigger registration") + return fmt.Errorf("failed to register triggers via coordinator: %w", err) + } + h.lggr.Infow("Registered triggers via coordinator", "workflowID", in.wid.Hex(), "triggerIDs", triggerIDs) + return nil +} + +// prepareEngineInputs decodes and validates the spec before any engine is +// built: the binary, owner and name must decode, and the workflow ID must +// match the one derived from the stored artifacts. +func (h *eventHandler) prepareEngineInputs(ctx context.Context, spec *job.WorkflowSpec) (*engineInputs, error) { // Ensure the capabilities registry is ready before creating any Engine instances. // This should be guaranteed by the Workflow Registry Syncer. if err := h.ensureCapRegistryReady(ctx); err != nil { - return fmt.Errorf("failed to ensure capabilities registry is ready: %w", err) + return nil, fmt.Errorf("failed to ensure capabilities registry is ready: %w", err) } decodedBinary, err := hex.DecodeString(spec.Workflow) if err != nil { - return nonRetryable(fmt.Errorf("failed to decode workflow spec binary: %w", err)) + return nil, nonRetryable(fmt.Errorf("failed to decode workflow spec binary: %w", err)) } // Free the hex-encoded binary string as it is not needed beyond this decode spec.Workflow = "" @@ -1166,88 +1350,91 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp // Workflow ID should match what is generated from the stored artifacts ownerBytes, err := hex.DecodeString(spec.WorkflowOwner) if err != nil { - return nonRetryable(fmt.Errorf("failed to decode owner: %w", err)) + return nil, nonRetryable(fmt.Errorf("failed to decode owner: %w", err)) } configBytes := []byte(spec.Config) hash, err := pkgworkflows.GenerateWorkflowID(ownerBytes, spec.WorkflowName, decodedBinary, configBytes, secretsURL) if err != nil { - return fmt.Errorf("failed to generate workflow id: %w", err) + return nil, fmt.Errorf("failed to generate workflow id: %w", err) } wid, err := types.WorkflowIDFromHex(spec.WorkflowID) if err != nil { - return nonRetryable(fmt.Errorf("invalid workflow id: %w", err)) + return nil, nonRetryable(fmt.Errorf("invalid workflow id: %w", err)) } if !types.WorkflowID(hash).Equal(wid) { - return nonRetryable(fmt.Errorf("workflowID mismatch: %s != %s", types.WorkflowID(hash).Hex(), wid.Hex())) + return nil, nonRetryable(fmt.Errorf("workflowID mismatch: %s != %s", types.WorkflowID(hash).Hex(), wid.Hex())) } - // Start a new WorkflowEngine instance, and add it to local engine registry workflowName, err := types.NewWorkflowName(spec.WorkflowName) if err != nil { - return nonRetryable(fmt.Errorf("invalid workflow name: %w", err)) + return nil, nonRetryable(fmt.Errorf("invalid workflow name: %w", err)) } - // Create a channel to receive the initialization result. - // This allows us to wait for the engine to complete initialization (including trigger subscriptions) - // before emitting the workflowActivated event, ensuring the event accurately reflects deployment status. - initDone := make(chan error, 1) - var engine services.Service - - engine, err = h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, configBytes, decodedBinary, spec.BinaryURL, initDone) + // Identity fingerprint stored alongside the engine in the registry. + reconcileKey, err := ReconcileKey(ownerBytes, spec.WorkflowName) if err != nil { - return fmt.Errorf("failed to create workflow engine: %w", err) + return nil, fmt.Errorf("failed to compute reconcile key: %w", err) } - if err = engine.Start(ctx); err != nil { + return &engineInputs{ + wid: wid, + workflowName: workflowName, + binary: decodedBinary, + config: configBytes, + reconcileKey: reconcileKey, + }, nil +} + +// startEngine starts engine and waits for it to complete initialization +// (including trigger subscriptions), so we don't emit workflowActivated events +// before the engine initializes successfully. The engine is closed if +// initialization fails or ctx is cancelled first. +func (h *eventHandler) startEngine(ctx context.Context, workflowID string, engine services.Service, initDone <-chan error) error { + if err := engine.Start(ctx); err != nil { return fmt.Errorf("failed to start workflow engine: %w", err) } - // Wait for the engine to complete initialization (including trigger subscriptions). - // This ensures we don't emit workflowActivated events before the engine initializes successfully. select { case <-ctx.Done(): // Context cancelled while waiting for initialization if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", spec.WorkflowID) + h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("context cancelled while waiting for engine initialization: %w", ctx.Err()) case initErr := <-initDone: if initErr != nil { // Engine initialization failed (e.g., trigger subscription failed) if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", spec.WorkflowID) + h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("engine initialization failed: %w", initErr) } } + return nil +} - // Engine is fully initialized, add to registry with source tracking and identity fingerprint - reconcileKey, err := ReconcileKey(ownerBytes, spec.WorkflowName) - if err != nil { - return fmt.Errorf("failed to compute reconcile key: %w", err) - } - if err := h.engineRegistry.AddWithReconcileKey(wid, source, reconcileKey, engine); err != nil { - if closeErr := engine.Close(); closeErr != nil { - return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) - } - - // Check for WorkflowID collision across sources - if errors.Is(err, ErrAlreadyExists) { - existingEntry, found := h.engineRegistry.Get(wid) - if found { - h.lggr.Warnw("WorkflowID collision detected: workflow already exists from different source", - "workflowID", wid.Hex(), - "attemptedSource", source, - "existingSource", existingEntry.Source, - "hint", "Each workflow ID should only be registered from a single source. Check your workflow configurations for duplicates.") - } +// registryAddFailed closes an engine the registry refused and returns the +// error to report. Both creation paths share it. +func (h *eventHandler) registryAddFailed(wid types.WorkflowID, source string, engine services.Service, err error) error { + if closeErr := engine.Close(); closeErr != nil { + return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) + } + + // Check for WorkflowID collision across sources + if errors.Is(err, ErrAlreadyExists) { + existingEntry, found := h.engineRegistry.Get(wid) + if found { + h.lggr.Warnw("WorkflowID collision detected: workflow already exists from different source", + "workflowID", wid.Hex(), + "attemptedSource", source, + "existingSource", existingEntry.Source, + "hint", "Each workflow ID should only be registered from a single source. Check your workflow configurations for duplicates.") } - - // This shouldn't happen because we call the handler serially and - // check for running engines above, see the call to engineRegistry.Contains. - return fmt.Errorf("invariant violation: %w", err) } - return nil + + // This shouldn't happen because we call the handler serially and + // check for running engines above, see the call to engineRegistry.Contains. + return fmt.Errorf("invariant violation: %w", err) } func (h *eventHandler) overrideFetcherForOwner(owner string) v2.SecretsFetcher { diff --git a/core/services/workflows/syncer/v2/handler_engine_create_test.go b/core/services/workflows/syncer/v2/handler_engine_create_test.go new file mode 100644 index 00000000000..50939ee7921 --- /dev/null +++ b/core/services/workflows/syncer/v2/handler_engine_create_test.go @@ -0,0 +1,276 @@ +package v2 + +import ( + "context" + "encoding/hex" + "errors" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace/noop" + + capreg "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" + "github.com/smartcontractkit/chainlink/v2/core/services/job" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" +) + +// fakeCoordinatedEngine is a v2.WorkflowEngine that reports as coordinated. +// Only the lifecycle methods tryEngineCreate calls are implemented. +type fakeCoordinatedEngine struct { + v2.WorkflowEngine + closed atomic.Bool +} + +func (f *fakeCoordinatedEngine) Start(context.Context) error { return nil } + +func (f *fakeCoordinatedEngine) Close() error { + f.closed.Store(true) + return nil +} + +func (f *fakeCoordinatedEngine) IsCoordinated() bool { return true } + +// recordingCoordinator records the subscribers handed to RegisterTriggers. +type recordingCoordinator struct { + TriggerCoordinator + registerErr error + registered []v2.Subscriber +} + +func (c *recordingCoordinator) RegisterTriggers(_ context.Context, subscriber v2.Subscriber, _ RegistrationParams) ([]string, error) { + c.registered = append(c.registered, subscriber) + return nil, c.registerErr +} + +// nodeConfig is the node state that decides the routing policy. +type nodeConfig struct { + flagOpen bool + withCoordinator bool + sharded bool +} + +// engineCreateFixture is an eventHandler whose engine factory mirrors the real +// one's contract: it reads the routing policy itself and returns the matching +// engine, leaving tryEngineCreate to route on the engine's IsCoordinated. +type engineCreateFixture struct { + h *eventHandler + legacyEngine v2.WorkflowEngine + coordinatedEngine *fakeCoordinatedEngine + factoryCalls atomic.Int32 + factoryErr error + coordinator *recordingCoordinator +} + +func newEngineCreateFixture(t *testing.T, cfg nodeConfig) *engineCreateFixture { + t.Helper() + lggr := logger.Test(t) + registry := capreg.NewRegistry(lggr) + registry.SetRegistryMetadata(&capreg.TestRegistryMetadata{}) + + f := &engineCreateFixture{ + legacyEngine: &mockEngine{}, + coordinatedEngine: &fakeCoordinatedEngine{}, + } + // A sharded node gets the ShardFailoverManager wrapper, as the real factory + // builds it. + if cfg.sharded { + f.legacyEngine = newTestShardFailoverManager(t, &mockEngine{}) + } + + f.h = &eventHandler{ + lggr: lggr, + capRegistry: registry, + engineRegistry: NewEngineRegistry(), + featureFlags: &v2.EngineFeatureFlags{CoordinatedEngine: limits.NewGateLimiter(cfg.flagOpen)}, + tracer: noop.NewTracerProvider().Tracer(""), + engineFactory: func(ctx context.Context, wfid, _ string, _ types.WorkflowName, _ string, _, _ []byte, _ string, initDone chan<- error) (v2.WorkflowEngine, error) { + f.factoryCalls.Add(1) + if f.factoryErr != nil { + return nil, f.factoryErr + } + initDone <- nil + if f.h.useCoordinatedEngine(ctx, wfid) { + return f.coordinatedEngine, nil + } + return f.legacyEngine, nil + }, + } + if cfg.withCoordinator { + f.coordinator = &recordingCoordinator{} + f.h.triggerCoordinator = f.coordinator + } + if cfg.sharded { + f.h.shardingEnabled = true + f.h.dispatcher = newFakeDispatcher() + } + return f +} + +// newTestWorkflowSpec returns a spec whose workflow ID matches its artifacts, +// so it passes prepareEngineInputs. +func newTestWorkflowSpec(t *testing.T) (*job.WorkflowSpec, types.WorkflowID) { + t.Helper() + owner := []byte{0x01, 0x02, 0x03} + name := "engine-create-test" + binary := []byte("binary") + config := []byte("config") + id, err := pkgworkflows.GenerateWorkflowID(owner, name, binary, config, "") + require.NoError(t, err) + wid := types.WorkflowID(id) + return &job.WorkflowSpec{ + Workflow: hex.EncodeToString(binary), + Config: string(config), + WorkflowID: wid.Hex(), + WorkflowOwner: hex.EncodeToString(owner), + WorkflowName: name, + }, wid +} + +func Test_tryEngineCreate_routing(t *testing.T) { + t.Parallel() + + t.Run("flag open with a coordinator takes the coordinated path", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: true, withCoordinator: true}) + spec, wid := newTestWorkflowSpec(t) + + require.NoError(t, f.h.tryEngineCreate(t.Context(), spec, "TestSource")) + + require.Equal(t, int32(1), f.factoryCalls.Load()) + entry, ok := f.h.engineRegistry.Get(wid) + require.True(t, ok) + require.True(t, entry.Coordinated()) + require.NotEmpty(t, entry.ReconcileKey) + require.Equal(t, []v2.Subscriber{f.coordinatedEngine}, f.coordinator.registered) + }) + + t.Run("flag closed takes the legacy path", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: false, withCoordinator: true}) + spec, wid := newTestWorkflowSpec(t) + + require.NoError(t, f.h.tryEngineCreate(t.Context(), spec, "TestSource")) + + require.Equal(t, int32(1), f.factoryCalls.Load()) + entry, ok := f.h.engineRegistry.Get(wid) + require.True(t, ok) + require.False(t, entry.Coordinated()) + require.Empty(t, f.coordinator.registered) + }) + + t.Run("flag open without a coordinator takes the legacy path", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: true, withCoordinator: false}) + spec, wid := newTestWorkflowSpec(t) + + require.NoError(t, f.h.tryEngineCreate(t.Context(), spec, "TestSource")) + + entry, ok := f.h.engineRegistry.Get(wid) + require.True(t, ok) + require.False(t, entry.Coordinated()) + }) + + t.Run("a sharded node stays on the legacy path with the flag open", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: true, withCoordinator: true, sharded: true}) + spec, wid := newTestWorkflowSpec(t) + + require.NoError(t, f.h.tryEngineCreate(t.Context(), spec, "TestSource")) + + entry, ok := f.h.engineRegistry.Get(wid) + require.True(t, ok) + require.IsType(t, &ShardFailoverManager{}, entry.Service) + require.False(t, entry.Coordinated(), "a sharded engine must never classify as coordinated") + require.Empty(t, f.coordinator.registered, "the coordinator must not be used on a sharded node") + require.Equal(t, engineCounts{legacy: 1}, countEngines(f.h.engineRegistry.GetAll())) + }) + + t.Run("RegisterTriggers failure removes and closes the coordinated engine", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: true, withCoordinator: true}) + f.coordinator.registerErr = errors.New("boom") + spec, wid := newTestWorkflowSpec(t) + + err := f.h.tryEngineCreate(t.Context(), spec, "TestSource") + + require.ErrorContains(t, err, "failed to register triggers via coordinator") + require.False(t, f.h.engineRegistry.Contains(wid)) + require.True(t, f.coordinatedEngine.closed.Load()) + }) + + t.Run("a factory failure is reported and registers nothing", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{withCoordinator: true}) + f.factoryErr = errors.New("boom") + spec, wid := newTestWorkflowSpec(t) + + err := f.h.tryEngineCreate(t.Context(), spec, "TestSource") + + require.ErrorContains(t, err, "failed to create workflow engine") + require.False(t, f.h.engineRegistry.Contains(wid)) + }) + + t.Run("invalid spec is rejected before any engine is built", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{withCoordinator: true}) + spec, _ := newTestWorkflowSpec(t) + spec.WorkflowID = types.WorkflowID{0xff}.Hex() + + err := f.h.tryEngineCreate(t.Context(), spec, "TestSource") + + require.ErrorContains(t, err, "workflowID mismatch") + require.Equal(t, int32(0), f.factoryCalls.Load()) + }) +} + +// Test_useCoordinatedEngine covers the routing policy the engine factory reads. +func Test_useCoordinatedEngine(t *testing.T) { + t.Parallel() + + newHandler := func(flagOpen, withCoordinator, sharded bool) *eventHandler { + h := &eventHandler{ + lggr: logger.Test(t), + featureFlags: &v2.EngineFeatureFlags{CoordinatedEngine: limits.NewGateLimiter(flagOpen)}, + } + if withCoordinator { + h.triggerCoordinator = &recordingCoordinator{} + } + if sharded { + h.shardingEnabled = true + h.dispatcher = newFakeDispatcher() + } + return h + } + + t.Run("flag open with a coordinator", func(t *testing.T) { + t.Parallel() + require.True(t, newHandler(true, true, false).useCoordinatedEngine(t.Context(), "wfid")) + }) + + t.Run("flag closed", func(t *testing.T) { + t.Parallel() + require.False(t, newHandler(false, true, false).useCoordinatedEngine(t.Context(), "wfid")) + }) + + t.Run("no coordinator wired", func(t *testing.T) { + t.Parallel() + require.False(t, newHandler(true, false, false).useCoordinatedEngine(t.Context(), "wfid")) + }) + + t.Run("sharded nodes stay on the legacy engine", func(t *testing.T) { + t.Parallel() + require.False(t, newHandler(true, true, true).useCoordinatedEngine(t.Context(), "wfid")) + }) + + t.Run("nil feature flags", func(t *testing.T) { + t.Parallel() + h := newHandler(true, true, false) + h.featureFlags = nil + require.False(t, h.useCoordinatedEngine(t.Context(), "wfid")) + }) +} diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index 4943d276db7..59146bafca7 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -65,6 +65,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncerlimiter" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2/triggers" ) type mockFetchResp struct { @@ -125,6 +126,22 @@ func (m *mockEngine) HealthReport() map[string]error { return nil } func (m *mockEngine) Name() string { return "mockEngine" } +func (m *mockEngine) ExecuteTrigger(context.Context, triggers.CoordinatedEvent) error { return nil } + +func (m *mockEngine) Subscribe(context.Context) ([]*sdk.TriggerSubscription, error) { + return nil, nil +} + +func (m *mockEngine) Tenant() contexts.CRE { return contexts.CRE{} } + +func (m *mockEngine) Drain() bool { return false } + +func (m *mockEngine) ActiveExecutions() int32 { return 0 } + +func (m *mockEngine) DrainStartedAt() (time.Time, bool) { return time.Time{}, false } + +func (m *mockEngine) IsCoordinated() bool { return false } + type mockDrainableEngine struct { mockEngine draining atomic.Bool @@ -162,7 +179,7 @@ func (m *mockDrainableEngine) Close() error { // mockEngineFactory returns a standard mock engine factory for tests. // It sends nil to initDone to signal successful initialization. -func mockEngineFactory(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) { +func mockEngineFactory(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) { if initDone != nil { initDone <- nil } @@ -317,7 +334,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { signedConfigURL: {Body: config, Err: nil}, }) }, - engineFactoryFn: func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) { + engineFactoryFn: func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) { if _, err := hex.DecodeString(name.Hex()); err != nil { return nil, fmt.Errorf("invalid workflow name: %w", err) } @@ -361,7 +378,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { signedConfigURL: {Body: config, Err: nil}, }) }, - engineFactoryFn: func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) { + engineFactoryFn: func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) { if initDone != nil { initDone <- nil } @@ -960,7 +977,7 @@ type testCase struct { fetcherFactory func(wfID []byte) *mockFetcher Event func(wfID []byte, wfName string, wfOwner []byte) WorkflowRegisteredEvent validationFn func(t *testing.T, ctx context.Context, event WorkflowRegisteredEvent, h *eventHandler, s *artifacts.Store, wfOwner []byte, wfName string, wfID types.WorkflowID, fetcher *mockFetcher, binaryURL, configURL string) - engineFactoryFn func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) + engineFactoryFn func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) } func testRunningWorkflow(t *testing.T, tc testCase) { diff --git a/core/services/workflows/syncer/v2/metrics.go b/core/services/workflows/syncer/v2/metrics.go index 9485daf2651..76f22eb9978 100644 --- a/core/services/workflows/syncer/v2/metrics.go +++ b/core/services/workflows/syncer/v2/metrics.go @@ -11,16 +11,27 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/beholder" ) +// engineType label values for the running-engines gauge. +const ( + engineTypeCoordinated = "coordinated" + engineTypeLegacy = "legacy" +) + type metrics struct { handleDuration metric.Int64Histogram fetchedWorkflows metric.Int64Gauge runningWorkflows metric.Int64Gauge drainingWorkflows metric.Int64Gauge - completedSyncs metric.Int64Counter - drainStarted metric.Int64Counter - drainCompleted metric.Int64Counter - drainDuration metric.Int64Histogram - deleteDeferred metric.Int64Counter + // runningEngines counts the engines currently in the registry, split by + // engineType ("coordinated" or "legacy"). Both series are always reported, + // including the zeroes, so an absent series means the node stopped + // reporting rather than "no engines of that type". + runningEngines metric.Int64Gauge + completedSyncs metric.Int64Counter + drainStarted metric.Int64Counter + drainCompleted metric.Int64Counter + drainDuration metric.Int64Histogram + deleteDeferred metric.Int64Counter // Per-source metrics for multi-source observability sourceHealth metric.Int64Gauge // 1=healthy, 0=unhealthy per source @@ -62,6 +73,19 @@ func (m *metrics) recordDrainingWorkflows(ctx context.Context, count int) { m.drainingWorkflows.Record(ctx, int64(count)) } +// recordRunningEngines reports the engine-type split of the registry. Called +// once per reconciliation tick, with both counts, so the two series stay in +// lockstep and "coordinated == 0" is an assertion the alert can rely on rather +// than the absence of a sample. +func (m *metrics) recordRunningEngines(ctx context.Context, coordinated, legacy int) { + m.runningEngines.Record(ctx, int64(coordinated), metric.WithAttributes( + attribute.String("engineType", engineTypeCoordinated), + )) + m.runningEngines.Record(ctx, int64(legacy), metric.WithAttributes( + attribute.String("engineType", engineTypeLegacy), + )) +} + func (m *metrics) incrementCompletedSyncs(ctx context.Context) { m.completedSyncs.Add(ctx, 1) } @@ -288,6 +312,11 @@ func newMetrics() (*metrics, error) { return nil, err } + runningEngines, err := beholder.GetMeter().Int64Gauge("platform_workflow_registry_syncer_running_engines") + if err != nil { + return nil, err + } + completedSyncs, err := beholder.GetMeter().Int64Counter("platform_workflow_registry_syncer_completed_syncs_total") if err != nil { return nil, err @@ -374,6 +403,7 @@ func newMetrics() (*metrics, error) { fetchedWorkflows: fetchedWorkflows, runningWorkflows: runningWorkflows, drainingWorkflows: drainingWorkflows, + runningEngines: runningEngines, completedSyncs: completedSyncs, drainStarted: drainStarted, drainCompleted: drainCompleted, diff --git a/core/services/workflows/syncer/v2/shard_failover_manager.go b/core/services/workflows/syncer/v2/shard_failover_manager.go index 987f359816e..dcf2b5116c8 100644 --- a/core/services/workflows/syncer/v2/shard_failover_manager.go +++ b/core/services/workflows/syncer/v2/shard_failover_manager.go @@ -8,9 +8,11 @@ import ( "time" commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" ringpb "github.com/smartcontractkit/chainlink-protos/ring/go" "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/sharding" @@ -25,6 +27,11 @@ import ( // cachedExpiry is how long a cached trigger event is kept for potential failover replay. const cachedExpiry = 10 * time.Minute +// ErrCoordinatedShardingUnsupported is returned by the coordinator-facing +// methods of ShardFailoverManager. Sharding and the coordinated engine are +// currently not compatible, so nothing should reach them. +var ErrCoordinatedShardingUnsupported = errors.New("coordinated engine is not supported on a sharded node") + // ShardFailoverManager wraps a workflow Engine and handles shard ownership // decisions externally, keeping the Engine oblivious to sharding. On the // primary shard it allows all trigger events through. On a secondary shard @@ -409,4 +416,29 @@ func (m *ShardFailoverManager) ActiveExecutions() int32 { return m.engine.Active // DrainStartedAt delegates to the wrapped engine. func (m *ShardFailoverManager) DrainStartedAt() (time.Time, bool) { return m.engine.DrainStartedAt() } -var _ DrainableService = (*ShardFailoverManager)(nil) +// ExecuteTrigger is unreachable on a sharded node: the coordinator is the only +// caller of the EventSink path, and a sharded node never takes the coordinated +// path. Failover replay calls the wrapped engine directly, not this method. +func (m *ShardFailoverManager) ExecuteTrigger(context.Context, triggers.CoordinatedEvent) error { + return ErrCoordinatedShardingUnsupported +} + +// Subscribe is unreachable on a sharded node: only the coordinator calls it, +// while registering triggers for a coordinated engine. +func (m *ShardFailoverManager) Subscribe(context.Context) ([]*sdkpb.TriggerSubscription, error) { + return nil, ErrCoordinatedShardingUnsupported +} + +// Tenant delegates to the wrapped engine. It has no error to return, and as a +// plain getter it cannot half-work, so it stays a passthrough. +func (m *ShardFailoverManager) Tenant() contexts.CRE { return m.engine.Tenant() } + +// IsCoordinated delegates to the wrapped engine. Unlike the methods above this +// is called on the legacy path — every reconciliation tick reads it to classify +// the engine — so it must answer rather than fail. +func (m *ShardFailoverManager) IsCoordinated() bool { return m.engine.IsCoordinated() } + +var ( + _ DrainableService = (*ShardFailoverManager)(nil) + _ v2.WorkflowEngine = (*ShardFailoverManager)(nil) +) diff --git a/core/services/workflows/syncer/v2/shard_failover_manager_test.go b/core/services/workflows/syncer/v2/shard_failover_manager_test.go index fb73d29a82d..eee0d0f8bf0 100644 --- a/core/services/workflows/syncer/v2/shard_failover_manager_test.go +++ b/core/services/workflows/syncer/v2/shard_failover_manager_test.go @@ -19,6 +19,8 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/sharding" remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2/triggers" ) // --- test doubles --- @@ -340,3 +342,49 @@ func (r *noopReceiver) Receive(context.Context, *remotetypes.MessageBody) {} // Ensure fakeDonNotifier satisfies the interface. var _ capabilities.DonSubscriber = (*fakeDonSubscriber)(nil) + +// newTestShardFailoverManager wraps engine in a manager that is never started, +// so the sharding collaborators are left nil. +func newTestShardFailoverManager(t *testing.T, engine v2.WorkflowEngine) *ShardFailoverManager { + t.Helper() + m := NewShardFailoverManager(ShardFailoverManagerConfig{ + ShardingEnabled: true, + WorkflowID: "wf-1", + WorkflowOwner: "0xowner", + Logger: logger.Test(t), + }) + m.SetEngine(engine) + return m +} + +// TestShardFailoverManager_DelegatesIsCoordinated covers the classification of +// the wrapper a sharded node registers, which must follow the engine it wraps. +func TestShardFailoverManager_DelegatesIsCoordinated(t *testing.T) { + t.Parallel() + + t.Run("wrapping a legacy engine reports legacy", func(t *testing.T) { + t.Parallel() + m := newTestShardFailoverManager(t, &mockEngine{}) + require.False(t, m.IsCoordinated()) + }) + + // Unreachable while sharding and the coordinated engine are mutually + // exclusive, but asserted so the value stays delegated rather than assumed. + t.Run("wrapping a coordinated engine reports coordinated", func(t *testing.T) { + t.Parallel() + m := newTestShardFailoverManager(t, &fakeCoordinatedDrainableEngine{}) + require.True(t, m.IsCoordinated()) + }) +} + +// TestShardFailoverManager_CoordinatorMethodsUnsupported covers the +// coordinator-facing methods, which must fail rather than delegate until +// sharding and the coordinated engine can work together. +func TestShardFailoverManager_CoordinatorMethodsUnsupported(t *testing.T) { + t.Parallel() + m := newTestShardFailoverManager(t, &mockEngine{}) + + _, err := m.Subscribe(t.Context()) + require.ErrorIs(t, err, ErrCoordinatedShardingUnsupported) + require.ErrorIs(t, m.ExecuteTrigger(t.Context(), triggers.CoordinatedEvent{}), ErrCoordinatedShardingUnsupported) +} diff --git a/core/services/workflows/syncer/v2/trigger_coordinator.go b/core/services/workflows/syncer/v2/trigger_coordinator.go new file mode 100644 index 00000000000..a4025e9b1e5 --- /dev/null +++ b/core/services/workflows/syncer/v2/trigger_coordinator.go @@ -0,0 +1,103 @@ +package v2 + +import ( + "context" + "errors" + + "github.com/jonboulle/clockwork" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" +) + +// ErrWorkflowNotCoordinated is returned by UnregisterTriggers for +// a workflowID the coordinator never registered. +var ErrWorkflowNotCoordinated = errors.New("workflow not registered with the trigger coordinator") + +// RegistrationParams is the per-workflow metadata RegisterTriggers needs to +// build each capability's RequestMetadata. +type RegistrationParams struct { + WorkflowOwner string + WorkflowName string + DecodedWorkflowName string + WorkflowTag string + WorkflowDonID uint32 + WorkflowDonConfigVersion uint32 + WorkflowRegistryChainSelector string + WorkflowRegistryAddress string +} + +// TriggerCoordinator owns trigger registration, the trigger handle map, event +// delivery, and acknowledgement for every workflow running the coordinated engine. +// It is a node-level singleton, started and stopped with the syncer. +type TriggerCoordinator interface { + services.Service + v2.Acknowledger + + // RegisterTriggers calls subscriber.Subscribe to obtain the engine's trigger + // subscriptions, registers them with the capability registry, retains the + // resulting handles and reader goroutines, and returns the registered + // trigger capability IDs. On any failure it unregisters what it already + // registered for this call and returns the error. + // Partial registration is never left behind. + RegisterTriggers(ctx context.Context, subscriber v2.Subscriber, params RegistrationParams) ([]string, error) + + // UnregisterTriggers stops ingress for workflowID immediately (unregisters with the capability registry) + // and cleans up the handle map once the engine has been drained and closed, so an execution already in flight + // can still resolve its handle to ACK. It also ensures that any resources associated with the workflow are properly released. + // This includes managing the workflowLimits by calling workflowLimits.Free. + // + // Returns ErrWorkflowNotCoordinated if workflowID was never registered here + UnregisterTriggers(workflowID string) error +} + +type noopTriggerCoordinator struct { + services.Service + eng *services.Engine + lggr logger.Logger +} + +// NewTriggerCoordinator returns the no-op coordinator: it logs the registration +// and teardown calls the syncer makes, and does nothing else. No trigger is +// registered with the capability registry and no event is ever delivered, so a +// workflow routed to the coordinated engine while this implementation is in +// place runs but never fires. +// +// capReg, engineRegistry and clock are unused here on purpose — the signature is +// the one the real coordinator needs, so wiring it does not churn call sites. +func NewTriggerCoordinator(capReg registry.CapabilitiesRegistry, engineRegistry *EngineRegistry, clock clockwork.Clock, lggr logger.Logger) TriggerCoordinator { + c := &noopTriggerCoordinator{ + lggr: logger.Named(lggr, "TriggerCoordinator"), + } + + c.Service, c.eng = services.Config{ + Name: "TriggerCoordinator", + Start: func(context.Context) error { + c.lggr.Warnw("No-op trigger coordinator started: workflows on the coordinated engine will register no triggers and receive no events") + return nil + }, + Close: func() error { return nil }, + }.NewServiceEngine(c.lggr) + + return c +} + +func (c *noopTriggerCoordinator) RegisterTriggers(ctx context.Context, subscriber v2.Subscriber, params RegistrationParams) ([]string, error) { + c.lggr.Infow("No-op RegisterTriggers", + "workflowID", subscriber.Tenant().Workflow, + "workflowOwner", params.WorkflowOwner, + "workflowName", params.DecodedWorkflowName, + "workflowDonID", params.WorkflowDonID) + return []string{}, nil +} + +func (c *noopTriggerCoordinator) Ack(ctx context.Context, triggerCapID, triggerRegistrationID, eventID string) error { + return nil +} + +func (c *noopTriggerCoordinator) UnregisterTriggers(workflowID string) error { + c.lggr.Infow("No-op UnregisterTriggers", "workflowID", workflowID) + return nil +} diff --git a/core/services/workflows/syncer/v2/workflow_registry.go b/core/services/workflows/syncer/v2/workflow_registry.go index 56143644606..d45ffc9a616 100644 --- a/core/services/workflows/syncer/v2/workflow_registry.go +++ b/core/services/workflows/syncer/v2/workflow_registry.go @@ -1075,22 +1075,42 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) runningWorkflows := w.engineRegistry.GetAll() w.metrics.recordRunningWorkflows(ctx, len(runningWorkflows)) - drainingWorkflows := 0 - for _, workflow := range runningWorkflows { - drainable, isDrainable := workflow.Service.(DrainableService) - if !isDrainable { - continue - } - if _, draining := drainable.DrainStartedAt(); draining { - drainingWorkflows++ - } - } - w.metrics.recordDrainingWorkflows(ctx, drainingWorkflows) + counts := countEngines(runningWorkflows) + w.metrics.recordDrainingWorkflows(ctx, counts.draining) + w.metrics.recordRunningEngines(ctx, counts.coordinated, counts.legacy) w.metrics.incrementCompletedSyncs(ctx) } } } +// engineCounts is the per-tick breakdown of the engines in the registry. +// coordinated + legacy always equals the number of engines; draining overlaps +// with both. +type engineCounts struct { + draining int + coordinated int + legacy int +} + +func countEngines(engines []ServiceWithMetadata) engineCounts { + var c engineCounts + for _, e := range engines { + if e.Coordinated() { + c.coordinated++ + } else { + c.legacy++ + } + drainable, isDrainable := e.Service.(DrainableService) + if !isDrainable { + continue + } + if _, draining := drainable.DrainStartedAt(); draining { + c.draining++ + } + } + return c +} + // reconcileOrphanedSpecs releases persisted specs whose workflow ID is absent // from all sources' metadata, catching workflows deleted while the node was // down — no engine exists at the next tick, so engine reconciliation can't diff --git a/core/services/workflows/syncer/v2/workflow_syncer_v2_test.go b/core/services/workflows/syncer/v2/workflow_syncer_v2_test.go index f6ad2d7d06c..e8b0d41be16 100644 --- a/core/services/workflows/syncer/v2/workflow_syncer_v2_test.go +++ b/core/services/workflows/syncer/v2/workflow_syncer_v2_test.go @@ -413,7 +413,7 @@ func Test_RegistrySyncer_WorkflowRegistered_InitiallyActivatedV2(t *testing.T) { require.NoError(t, err) handler, err := NewEventHandler(lggr, wfStore, nil, true, capRegistry, &confidentialrelay.ExecutionHandlers{}, er, - emitter, limiters, featureFlags, rl, wl, store, workflowEncryptionKey, donNotifier, WithStaticEngine(&mockService{}), withTestOrgResolver()) + emitter, limiters, featureFlags, rl, wl, store, workflowEncryptionKey, donNotifier, WithStaticEngine(&mockEngine{}), withTestOrgResolver()) require.NoError(t, err) worker, err := NewWorkflowRegistry( diff --git a/core/services/workflows/v2/config.go b/core/services/workflows/v2/config.go index bf993076a24..8470792fbee 100644 --- a/core/services/workflows/v2/config.go +++ b/core/services/workflows/v2/config.go @@ -12,7 +12,6 @@ import ( commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry" "github.com/smartcontractkit/chainlink-common/pkg/config" - "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -387,6 +386,10 @@ type EngineFeatureFlags struct { // which lets ops schedule a healing window across the DON via cresettings. // Nil when construction fails; call sites must nil-check. WorkflowTagBackfill limits.RangeLimiter[config.Timestamp] + + // CoordinatedEngine selects the coordinatedEngine over the legacy trigger-owning + // Engine for newly created workflows. Nil when construction fails; call sites must nil-check. + CoordinatedEngine limits.GateLimiter } func NewFeatureFlags(lf limits.Factory, cfgFn func(*cresettings.Workflows)) (*EngineFeatureFlags, error) { @@ -398,8 +401,14 @@ func NewFeatureFlags(lf limits.Factory, cfgFn func(*cresettings.Workflows)) (*En if err != nil { return nil, fmt.Errorf("workflow tag backfill flag: %w", err) } + + coordinatedEngine, err := limits.MakeGateLimiter(lf, cresettings.Default.CoordinatedEngineEnabled) + if err != nil { + return nil, fmt.Errorf("coordinated engine flag: %w", err) + } return &EngineFeatureFlags{ WorkflowTagBackfill: workflowTagBackfill, + CoordinatedEngine: coordinatedEngine, }, nil } @@ -427,12 +436,6 @@ type LifecycleHooks struct { // has completed initialization. It is also helpful for testing. OnInitialized func(err error) - // OnSubscriptionsReady is called after the WASM Subscribe call returns - // and the subscriptions have been validated, but before trigger - // registration begins. It allows the caller (syncer/dispatcher) to - // inspect or modify the subscriptions before they are registered with - // the capabilities registry. Returning an error aborts initialization. - OnSubscriptionsReady func(subs []*sdkpb.TriggerSubscription, cre contexts.CRE) error OnSubscribedToTriggers func(triggerIDs []string) OnTriggerEventDropped func(triggerID, eventID, reason string) OnExecutionFinished func(executionID string, status string) @@ -522,9 +525,6 @@ func (h *LifecycleHooks) setDefaultHooks() { if h.OnInitialized == nil { h.OnInitialized = func(err error) {} } - if h.OnSubscriptionsReady == nil { - h.OnSubscriptionsReady = func(subs []*sdkpb.TriggerSubscription, cre contexts.CRE) error { return nil } - } if h.OnSubscribedToTriggers == nil { h.OnSubscribedToTriggers = func(triggerIDs []string) {} } diff --git a/core/services/workflows/v2/engine_test.go b/core/services/workflows/v2/engine_test.go index d15b3eb0991..3c0c417cfb2 100644 --- a/core/services/workflows/v2/engine_test.go +++ b/core/services/workflows/v2/engine_test.go @@ -2900,10 +2900,6 @@ func newTestEngine( OnInitialized: func(err error) { e.initializedCh <- err }, - OnSubscriptionsReady: func(_ []*sdkpb.TriggerSubscription, _ contexts.CRE) error { - e.subscriptionsReadyCalls.Add(1) - return nil - }, OnSubscribedToTriggers: func(triggerIDs []string) { e.subscribedToTriggersCh <- triggerIDs }, diff --git a/deployment/go.mod b/deployment/go.mod index 1f743e39fc0..f5a7c04827a 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -46,7 +46,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 diff --git a/deployment/go.sum b/deployment/go.sum index 696a28f2166..52f18627293 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1403,8 +1403,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= diff --git a/go.mod b/go.mod index 946e7aae56f..cd444012002 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 diff --git a/go.sum b/go.sum index f258908d089..becf0382fdc 100644 --- a/go.sum +++ b/go.sum @@ -1104,8 +1104,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62/go.mod h1:c++Ed0jeYn4w8WIfiAboRwN6ClRRVGglL0qwB/QjJvM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 1315fb71ccb..8e5e92048ac 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -34,7 +34,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index cd4fa742768..518fda044e2 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1392,8 +1392,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 52186f75bce..5a3304cd3ed 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -24,7 +24,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.8 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 693a7d426a1..5beaa14a9e4 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1630,8 +1630,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index f1a412b4137..eec5705dd53 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -37,7 +37,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 893a2e27e16..b3d0f8dd751 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1654,8 +1654,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 5e260a91d1d..d0a1d40281e 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -76,7 +76,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index ee2dbf59075..43d50985161 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1833,8 +1833,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= From abf08375e7630761d012366e2522ca496657f294 Mon Sep 17 00:00:00 2001 From: amit-momin <108959691+amit-momin@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:10:18 +0000 Subject: [PATCH 5/8] Bump cl-evm (#23854) * Bumped cl-evm * Fixed docs tests * Added changeset --- .changeset/flat-worms-teach.md | 5 + core/config/docs/chains-evm.toml | 3 + core/scripts/go.mod | 2 +- core/scripts/go.sum | 4 +- core/services/chainlink/config_test.go | 3 + deployment/go.mod | 2 +- deployment/go.sum | 4 +- docs/CONFIG.md | 247 +++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- plugins/plugins.public.yaml | 2 +- system-tests/lib/go.mod | 2 +- system-tests/lib/go.sum | 4 +- system-tests/tests/go.mod | 2 +- system-tests/tests/go.sum | 4 +- 19 files changed, 280 insertions(+), 22 deletions(-) create mode 100644 .changeset/flat-worms-teach.md diff --git a/.changeset/flat-worms-teach.md b/.changeset/flat-worms-teach.md new file mode 100644 index 00000000000..a982cf507e0 --- /dev/null +++ b/.changeset/flat-worms-teach.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +Added feature to the EVM BalanceMonitor to allow tracking balance of ERC-20 type fee tokens #added diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 8292efd61b5..acaa5c3c9e1 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -191,6 +191,9 @@ FeeBoost = false # Default [EVM.BalanceMonitor] # Enabled balance monitoring for all keys. Enabled = true # Default +# ERC20TokenAddress is the address of an ERC-20 type token that a chain supports as a fee token. +ERC20TokenAddress = "0x20C0000000000000000000000000000000000000" # Example + [EVM.GasEstimator] # Mode controls what type of gas estimator is used. diff --git a/core/scripts/go.mod b/core/scripts/go.mod index a720949d3df..79da2f35895 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -36,7 +36,7 @@ require ( github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.8 diff --git a/core/scripts/go.sum b/core/scripts/go.sum index fa9116d6059..3874076ba55 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1695,8 +1695,8 @@ github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba4 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 125855c7f83..407c4735d39 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -1394,6 +1394,9 @@ func TestConfig_full(t *testing.T) { if got.EVM[c].GasEstimator.SenderAddress == nil { got.EVM[c].GasEstimator.SenderAddress = new(types.EIP55Address) } + if got.EVM[c].BalanceMonitor.ERC20TokenAddress == nil { + got.EVM[c].BalanceMonitor.ERC20TokenAddress = new(types.EIP55Address) + } } configtest.AssertFieldsNotNil(t, got) diff --git a/deployment/go.mod b/deployment/go.mod index f5a7c04827a..57475d24166 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -50,7 +50,7 @@ require ( github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260916140808-f5d22b3cb3ee diff --git a/deployment/go.sum b/deployment/go.sum index 52f18627293..520769a748d 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1413,8 +1413,8 @@ github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba4 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/docs/CONFIG.md b/docs/CONFIG.md index f25b19bb1ee..9c602d1dfa3 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -9228,6 +9228,126 @@ AcceptanceTimeout = '30s'

+
Tempo Mainnet (4217)

+ +```toml +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 50 +SafeDepth = 0 +FinalityTagEnabled = true +SafeTagSupported = true +LogBackfillBatchSize = 1000 +LogPollInterval = '1s' +LogPollerSkipEmptyBlocks = false +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 1 +MinContractPayment = '0.00001 link' +NonceAutoSync = true +NoNewHeadsThreshold = '3m0s' +LogBroadcasterEnabled = false +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 2 +FinalizedBlockOffset = 0 +NoNewFinalizedHeadsThreshold = '0s' + +[Transactions] +Enabled = true +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '1m0s' +ConfirmationTimeout = '1m0s' + +[Transactions.AutoPurge] +Enabled = false + +[Transactions.TransactionManagerV2] +Enabled = false + +[BalanceMonitor] +Enabled = true +ERC20TokenAddress = '0x20C0000000000000000000000000000000000000' + +[GasEstimator] +Mode = 'FeeHistory' +PriceDefault = '1 gwei' +PriceMax = '1 micro' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '1' +LimitTransfer = 21000 +EstimateLimit = false +BumpMin = '5 gwei' +BumpPercent = 20 +BumpThreshold = 30 +EIP1559DynamicFees = false +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 8 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 60 + +[GasEstimator.FeeHistory] +CacheTimeout = '4s' + +[HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = false +PersistenceEnabled = false +PersistenceBatchSize = 100 + +[NodePool] +PollFailureThreshold = 5 +PollSuccessThreshold = 0 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 15 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' +FinalizedStateCheckFailureThreshold = 0 +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' +NewHeadsPollInterval = '1s' +VerifyChainID = true +ExternalRequestMaxResponseSize = 1000000 + +[NodePool.Errors] +FinalizedStateUnavailable = '(: |^)(missing trie node|state not available|historical state unavailable)' + +[OCR] +ContractConfirmations = 1 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[Workflow] +GasLimitDefault = 400000 +TxAcceptanceState = 2 +PollPeriod = '2s' +AcceptanceTimeout = '30s' +``` + +

+
Worldchain Testnet (4801)

```toml @@ -10832,6 +10952,126 @@ AcceptanceTimeout = '30s'

+
Tempo Testnet Moderato (42431)

+ +```toml +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 50 +SafeDepth = 0 +FinalityTagEnabled = true +SafeTagSupported = true +LogBackfillBatchSize = 1000 +LogPollInterval = '1s' +LogPollerSkipEmptyBlocks = false +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 1 +MinContractPayment = '0.00001 link' +NonceAutoSync = true +NoNewHeadsThreshold = '3m0s' +LogBroadcasterEnabled = false +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 2 +FinalizedBlockOffset = 0 +NoNewFinalizedHeadsThreshold = '0s' + +[Transactions] +Enabled = true +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '1m0s' +ConfirmationTimeout = '1m0s' + +[Transactions.AutoPurge] +Enabled = false + +[Transactions.TransactionManagerV2] +Enabled = false + +[BalanceMonitor] +Enabled = true +ERC20TokenAddress = '0x20C0000000000000000000000000000000000000' + +[GasEstimator] +Mode = 'FeeHistory' +PriceDefault = '1 gwei' +PriceMax = '1 micro' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '4' +LimitTransfer = 21000 +EstimateLimit = false +BumpMin = '5 gwei' +BumpPercent = 20 +BumpThreshold = 30 +EIP1559DynamicFees = false +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 8 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 60 + +[GasEstimator.FeeHistory] +CacheTimeout = '4s' + +[HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = false +PersistenceEnabled = false +PersistenceBatchSize = 100 + +[NodePool] +PollFailureThreshold = 5 +PollSuccessThreshold = 0 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 15 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' +FinalizedStateCheckFailureThreshold = 0 +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' +NewHeadsPollInterval = '1s' +VerifyChainID = true +ExternalRequestMaxResponseSize = 1000000 + +[NodePool.Errors] +FinalizedStateUnavailable = '(: |^)(missing trie node|state not available|historical state unavailable)' + +[OCR] +ContractConfirmations = 1 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[Workflow] +GasLimitDefault = 400000 +TxAcceptanceState = 2 +PollPeriod = '2s' +AcceptanceTimeout = '30s' +``` + +

+
Hemi Mainnet (43111)

```toml @@ -16403,6 +16643,7 @@ FeeBoost enables node to immediately boost txs to the max gas price configured ```toml [EVM.BalanceMonitor] Enabled = true # Default +ERC20TokenAddress = "0x20C0000000000000000000000000000000000000" # Example ``` @@ -16412,6 +16653,12 @@ Enabled = true # Default ``` Enabled balance monitoring for all keys. +### ERC20TokenAddress +```toml +ERC20TokenAddress = "0x20C0000000000000000000000000000000000000" # Example +``` +ERC20TokenAddress is the address of an ERC-20 type token that a chain supports as a fee token. + ## EVM.GasEstimator ```toml [EVM.GasEstimator] diff --git a/go.mod b/go.mod index cd444012002..87185644328 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ require ( github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260710181111-6417709a55ee github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 diff --git a/go.sum b/go.sum index becf0382fdc..e891a1bb9d3 100644 --- a/go.sum +++ b/go.sum @@ -1112,8 +1112,8 @@ github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.202609151 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260710181111-6417709a55ee h1:MfUoOYDYeE4vzk2FXDw65V0CjDlVczYd5ohCBRwByjk= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 8e5e92048ac..78edd2fd3ec 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -37,7 +37,7 @@ require ( github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea github.com/smartcontractkit/chainlink-sui v0.0.0-20260922143238-46cc062ae5a3 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 518fda044e2..03415ae1343 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1402,8 +1402,8 @@ github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba4 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 5a3304cd3ed..db655a5b95e 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -26,7 +26,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.8 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.5 github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 5beaa14a9e4..e2b72f32542 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1640,8 +1640,8 @@ github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba4 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/plugins/plugins.public.yaml b/plugins/plugins.public.yaml index e1dd66f2429..8ef01e827bf 100644 --- a/plugins/plugins.public.yaml +++ b/plugins/plugins.public.yaml @@ -58,7 +58,7 @@ plugins: evm: - moduleURI: "github.com/smartcontractkit/chainlink-evm" - gitRef: "v0.3.4-0.20260917132551-2a2b38e8b744" + gitRef: "v0.3.4-0.20260929161505-cc645fce4250" installPath: "./pkg/cmd/chainlink-evm" capability-cron: diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index eec5705dd53..6ad7ead9188 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -41,7 +41,7 @@ require ( github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260916140808-f5d22b3cb3ee github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index b3d0f8dd751..8567ffa90d4 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1666,8 +1666,8 @@ github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba4 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index d0a1d40281e..c40d721d96b 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -625,7 +625,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 // indirect + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260723212800-b2f21d31b1d2 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 43d50985161..a23aed2d89a 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1847,8 +1847,8 @@ github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba4 github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= From ef5367f73928254d6d85a69e8eed332012f9cef5 Mon Sep 17 00:00:00 2001 From: Bolek <1416262+bolekk@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:16:56 +0000 Subject: [PATCH 6/8] fix: eliminate race in TestTriggerPublisher_ExplicitUnregister (#23853) RegisterTrigger runs asynchronously; waitForRegistration only confirms the call reached the underlying mock, not that the publisher has recorded the registration in its internal map. If the unregister message won that race, the handler saw the trigger as unregistered and dropped it, causing the test to time out waiting on waitForUnregister. Wait for a registration check (only sent for triggers already in the registration map) before issuing the unregister, closing the window. Co-authored-by: Claude Sonnet 5 --- .../remote/trigger_publisher_test.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/core/capabilities/remote/trigger_publisher_test.go b/core/capabilities/remote/trigger_publisher_test.go index 90d77b85bc1..5849aabfe71 100644 --- a/core/capabilities/remote/trigger_publisher_test.go +++ b/core/capabilities/remote/trigger_publisher_test.go @@ -522,7 +522,13 @@ func TestTriggerPublisher_ExplicitUnregister(t *testing.T) { } dispatcher := mocks.NewDispatcher(t) - allowRegistrationChecks(dispatcher) + checkReceived := make(chan *remotetypes.MessageBody, 10) + dispatcher.On("Send", mock.Anything, mock.MatchedBy(func(m *remotetypes.MessageBody) bool { + return m.Method == remotetypes.MethodTriggerRegistrationCheck + })).Run(func(args mock.Arguments) { + msg := args.Get(1).(*remotetypes.MessageBody) + checkReceived <- msg + }).Return(nil).Maybe() config := &commoncap.RemoteTriggerConfig{ RegistrationRefresh: 100 * time.Millisecond, @@ -549,6 +555,17 @@ func TestTriggerPublisher_ExplicitUnregister(t *testing.T) { waitForRegistration(t, underlying.registrationsCh) + // RegisterTrigger on the underlying capability runs asynchronously; the publisher only + // records the registration (and becomes eligible to process an unregister for it) after + // that call returns. Wait for a registration check, which is only sent for triggers + // already present in the publisher's registration map, so the race below can't drop the + // unregister message. + select { + case <-checkReceived: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for registration check message") + } + // Send unregister unregMsg := &remotetypes.MessageBody{ Sender: peers[1][:], From 219e22449238c320e152d051a2d0991ee925bed9 Mon Sep 17 00:00:00 2001 From: Cedric Date: Tue, 29 Sep 2026 17:19:58 +0000 Subject: [PATCH 7/8] Bump consensus capability (#23855) --- plugins/plugins.public.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugins.public.yaml b/plugins/plugins.public.yaml index 8ef01e827bf..06ea33bfaa6 100644 --- a/plugins/plugins.public.yaml +++ b/plugins/plugins.public.yaml @@ -69,7 +69,7 @@ plugins: capability-consensus: - moduleURI: "github.com/smartcontractkit/capabilities/consensus" - gitRef: "2aaae97ddf6900b082391c4756687b7584bfaf8a" + gitRef: "968e708eec352f300465080cbbec49dff36aa488" installPath: "." capability-httpaction: From 85e1d5c16a5a77076446e0430a939f0c347a7b97 Mon Sep 17 00:00:00 2001 From: Russell Stern Date: Tue, 29 Sep 2026 17:37:02 +0000 Subject: [PATCH 8/8] Speedup vault tests (#23847) * Switch test workflow to use http instead of cron * Cache compiled workflows * Fixed go mod * Revert timeout back to 7 minutes * Cleaned up comments --- .github/workflows/cre-system-tests.yaml | 4 +- go.md | 2 +- system-tests/lib/cre/workflow/compile.go | 183 +++++++++++- .../tests/smoke/cre/vault_don_test.go | 177 ++++++------ .../tests/smoke/cre/vault_don_test_helpers.go | 262 ++++++++++++++---- .../smoke/cre/vaultsecret/config/config.go | 72 ++--- .../tests/smoke/cre/vaultsecret/go.mod | 2 +- .../tests/smoke/cre/vaultsecret/go.sum | 4 +- .../tests/smoke/cre/vaultsecret/main.go | 67 +++-- 9 files changed, 545 insertions(+), 228 deletions(-) diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml index 89daed46fc1..1a59dd0ff94 100644 --- a/.github/workflows/cre-system-tests.yaml +++ b/.github/workflows/cre-system-tests.yaml @@ -190,7 +190,7 @@ jobs: # http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments name: integration deployment: false - timeout-minutes: 13 + timeout-minutes: 10 env: ENABLE_AUTO_QUARANTINE: "true" BILLING_PLATFORM_SERVICE_IMAGE: @@ -415,7 +415,7 @@ jobs: continue-on-error: ${{ env.ENABLE_AUTO_QUARANTINE == 'true' }} env: TEST_NAME: ${{ matrix.tests.test_name }} - TEST_TIMEOUT: 10m # let's leave 3 minutes for other steps (the whole job times out after 13 minutes) + TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes) RUN_QUARANTINED_TESTS: "true" # always run quarantined tests in CI TOPOLOGY_NAME: ${{ matrix.tests.topology }} CONFIDENTIAL_COMPUTE_ROOT: ${{ github.workspace }}/chainlink-confidential-compute diff --git a/go.md b/go.md index 45e9aaf0b6a..a645767a478 100644 --- a/go.md +++ b/go.md @@ -593,7 +593,7 @@ flowchart LR chainlink/system-tests/tests/smoke/cre/stellar/stellarwrite --> cre-sdk-go/capabilities/blockchain/stellar chainlink/system-tests/tests/smoke/cre/stellar/stellarwrite --> cre-sdk-go/capabilities/scheduler/cron click chainlink/system-tests/tests/smoke/cre/stellar/stellarwrite href "https://github.com/smartcontractkit/chainlink" - chainlink/system-tests/tests/smoke/cre/vaultsecret --> cre-sdk-go/capabilities/scheduler/cron + chainlink/system-tests/tests/smoke/cre/vaultsecret --> cre-sdk-go/capabilities/networking/http click chainlink/system-tests/tests/smoke/cre/vaultsecret href "https://github.com/smartcontractkit/chainlink" chainlink/v2 --> chainlink-aptos/codec chainlink/v2 --> chainlink-ccv diff --git a/system-tests/lib/cre/workflow/compile.go b/system-tests/lib/cre/workflow/compile.go index fea3d162ac6..443f598e7d4 100644 --- a/system-tests/lib/cre/workflow/compile.go +++ b/system-tests/lib/cre/workflow/compile.go @@ -3,15 +3,22 @@ package workflow import ( "bytes" "context" + "crypto/sha256" "encoding/base64" + "encoding/hex" "fmt" + "io/fs" "os" "os/exec" "path/filepath" + "sort" "strings" + "sync" "github.com/andybalholm/brotli" "github.com/pkg/errors" + + "github.com/smartcontractkit/chainlink-testing-framework/framework" ) type Language = string @@ -21,6 +28,9 @@ const ( LanguageTS Language = "typescript" ) +// Set CRE_TEST_COMPILE_CACHE_DISABLED=true to bypass the in-process compile cache. +const compileCacheDisabledEnv = "CRE_TEST_COMPILE_CACHE_DISABLED" + // CompileWorkflow compiles a workflow from a file path (absolute or relative) and returns the path to the compiled workflow. // workflowFilePath is the path to the workflow file. // workflowName is the name of the workflow. @@ -33,6 +43,8 @@ func CompileWorkflow(ctx context.Context, workflowFilePath, workflowName string) // CompileWorkflowToDir compiles a workflow and stores build artifacts in outputDir. // If outputDir is empty, a temporary directory is created automatically. +// Artifacts are cached in-process by a hash of the workflow directory, so +// identical sources compile once per test run. func CompileWorkflowToDir(ctx context.Context, workflowFilePath, workflowName, outputDir string) (string, error) { if len(workflowName) < 10 { return "", errors.New("workflow name must be at least 10 characters long") @@ -48,6 +60,175 @@ func CompileWorkflowToDir(ctx context.Context, workflowFilePath, workflowName, o return "", errors.Wrap(mkErr, "failed to prepare workflow build dir") } + if compileCacheEnabled() { + key, keyErr := workflowSourceCacheKey(workflowFilePath) + if keyErr == nil { + return compileWorkflowToDirCached(ctx, key, workflowFilePath, workflowName, outputDir) + } + framework.L.Warn().Err(keyErr).Str("workflow_file", workflowFilePath).Msg("failed to hash workflow sources; compiling without the compile cache") + } + + return compileWorkflowToDirUncached(ctx, workflowFilePath, workflowName, outputDir) +} + +type compileCacheEntry struct { + done chan struct{} + artifact []byte + err error +} + +var ( + compileCacheMu sync.Mutex + compileCache = map[string]*compileCacheEntry{} +) + +func compileCacheEnabled() bool { + return strings.TrimSpace(strings.ToLower(os.Getenv(compileCacheDisabledEnv))) != "true" +} + +// getOrCreateCompileCacheEntry returns the entry for key and whether the +// caller is the leader that must compile it. +func getOrCreateCompileCacheEntry(key string) (entry *compileCacheEntry, leader bool) { + compileCacheMu.Lock() + defer compileCacheMu.Unlock() + if existing, ok := compileCache[key]; ok { + return existing, false + } + entry = &compileCacheEntry{done: make(chan struct{})} + compileCache[key] = entry + return entry, true +} + +// finalizeCompileCacheEntry publishes the result to waiters. Failed entries +// are evicted so later callers retry instead of reusing the failure. +func finalizeCompileCacheEntry(key string, entry *compileCacheEntry, artifact []byte, err error) { + entry.artifact = artifact + entry.err = err + compileCacheMu.Lock() + defer compileCacheMu.Unlock() + if err != nil { + delete(compileCache, key) + } + close(entry.done) +} + +func compileWorkflowToDirCached(ctx context.Context, cacheKey, workflowFilePath, workflowName, outputDir string) (string, error) { + for { + entry, leader := getOrCreateCompileCacheEntry(cacheKey) + if leader { + return compileAsCompileCacheLeader(ctx, cacheKey, entry, workflowFilePath, workflowName, outputDir) + } + + if err := waitForCompileCacheEntry(ctx, entry); err != nil { + // The failed entry was evicted, so retrying makes this caller the + // leader (covers a previous leader whose context was cancelled). + framework.L.Warn().Str("workflow_name", workflowName).Err(err).Msg("in-flight workflow compile failed; retrying as the compile leader") + continue + } + + framework.L.Info().Str("workflow_name", workflowName).Str("workflow_file", workflowFilePath).Msg("workflow compile cache hit; reusing compiled workflow artifact") + return writeCompressedArtifact(outputDir, workflowName, entry.artifact) + } +} + +func compileAsCompileCacheLeader(ctx context.Context, cacheKey string, entry *compileCacheEntry, workflowFilePath, workflowName, outputDir string) (string, error) { + framework.L.Info().Str("workflow_name", workflowName).Str("workflow_file", workflowFilePath).Msg("workflow compile cache miss; compiling workflow") + + artifactPath, compileErr := compileWorkflowToDirUncached(ctx, workflowFilePath, workflowName, outputDir) + + var artifact []byte + cacheErr := compileErr + if cacheErr == nil { + b, readErr := os.ReadFile(artifactPath) + if readErr != nil { + // Artifact is still valid for this caller; only caching is skipped. + cacheErr = errors.Wrap(readErr, "failed to read compiled workflow artifact for the compile cache") + } else { + artifact = b + } + } + + finalizeCompileCacheEntry(cacheKey, entry, artifact, cacheErr) + return artifactPath, compileErr +} + +// waitForCompileCacheEntry blocks until the leader finishes or ctx ends. +func waitForCompileCacheEntry(ctx context.Context, entry *compileCacheEntry) error { + select { + case <-entry.done: + if entry.err != nil { + return errors.Wrap(entry.err, "in-flight workflow compile failed") + } + if len(entry.artifact) == 0 { + return errors.New("in-flight workflow compile produced no artifact") + } + return nil + case <-ctx.Done(): + return errors.Wrap(ctx.Err(), "cancelled while waiting for an in-flight workflow compile") + } +} + +// workflowSourceCacheKey hashes all build inputs in the workflow's directory +// (sources, go.mod/go.sum, package.json, ...), excluding build outputs. +func workflowSourceCacheKey(workflowFilePath string) (string, error) { + dir := filepath.Dir(workflowFilePath) + + var files []string + walkErr := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == ".git" || d.Name() == "node_modules" { + return filepath.SkipDir + } + return nil + } + if strings.HasSuffix(d.Name(), ".wasm") || strings.HasSuffix(d.Name(), ".br.b64") { + return nil + } + files = append(files, path) + return nil + }) + if walkErr != nil { + return "", errors.Wrapf(walkErr, "failed to walk workflow source dir %s", dir) + } + + sort.Strings(files) + hasher := sha256.New() + for _, path := range files { + rel, relErr := filepath.Rel(dir, path) + if relErr != nil { + return "", errors.Wrap(relErr, "failed to relativize workflow source path") + } + content, readErr := os.ReadFile(path) + if readErr != nil { + return "", errors.Wrapf(readErr, "failed to read workflow source file %s", path) + } + hasher.Write([]byte(rel)) + hasher.Write([]byte{0}) + hasher.Write(content) + hasher.Write([]byte{0}) + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +// writeCompressedArtifact writes a cached artifact to +// outputDir/.br.b64, matching compressWorkflow's naming. +func writeCompressedArtifact(outputDir, workflowName string, artifact []byte) (string, error) { + if mkErr := os.MkdirAll(outputDir, 0o755); mkErr != nil { + return "", errors.Wrap(mkErr, "failed to prepare workflow build dir") + } + outputFile := filepath.Join(outputDir, workflowName+".br.b64") + if writeErr := os.WriteFile(outputFile, artifact, 0o644); writeErr != nil { //nolint:gosec // G306: we want it to be readable by everyone, same as compressWorkflow + return "", errors.Wrap(writeErr, "failed to write cached workflow artifact") + } + return outputFile, nil +} + +// compileWorkflowToDirUncached compiles the workflow and returns the path to +// the compressed artifact in outputDir. +func compileWorkflowToDirUncached(ctx context.Context, workflowFilePath, workflowName, outputDir string) (string, error) { language, lErr := delectLanguage(workflowFilePath) if lErr != nil { return "", errors.Wrap(lErr, "failed to detect workflow language") @@ -162,7 +343,7 @@ func compressWorkflow(workflowWasmPath string) (string, error) { } if err := os.WriteFile(outputFile, outputData, 0644); err != nil { //nolint:gosec // G306: we want it to be readable by everyone - return "", errors.Wrap(err, "failed to write output file") + return "", errors.Wrap(err, "failed to write compressed output file") } outputFileAbsPath, outputFileAbsPathErr := filepath.Abs(outputFile) diff --git a/system-tests/tests/smoke/cre/vault_don_test.go b/system-tests/tests/smoke/cre/vault_don_test.go index 0b2908104e0..664e50662ed 100644 --- a/system-tests/tests/smoke/cre/vault_don_test.go +++ b/system-tests/tests/smoke/cre/vault_don_test.go @@ -14,37 +14,32 @@ import ( "testing" "time" + retry "github.com/avast/retry-go/v4" "github.com/ethereum/go-ethereum/common" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" - retry "github.com/avast/retry-go/v4" - vault_helpers "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" + workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common" workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events" + "github.com/smartcontractkit/chainlink-testing-framework/framework" + "github.com/smartcontractkit/chainlink-testing-framework/seth" keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm" - t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" - vaultcap "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils" - - workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" - envconfig "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/vault" - ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" - - "github.com/smartcontractkit/chainlink-testing-framework/framework" - "github.com/smartcontractkit/chainlink-testing-framework/seth" - creworkflow "github.com/smartcontractkit/chainlink/system-tests/lib/cre/workflow" vaultsecret_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/vaultsecret/config" + t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" + ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" + vaultcap "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils" ) // ExecuteVaultAllowListBasedTests covers vault gateway + workflows with allow-listed JSON-RPC auth @@ -62,6 +57,10 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture gwURL := fixture.GatewayURL.String() vaultPublicKey := fixture.VaultPublicKey + // Shared by all subtests; checks are passed per HTTP trigger. Safe for the + // parallel identifier_validation subtest since its checks are negative-only. + verifier := deployVaultVerifierWorkflow(t, testEnv, fixture.TriggerAuth, "allowlist-verifier") + t.Run("allowlist_delete_batch_at_limit", func(t *testing.T) { sc := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient owner := sc.MustGetRootKeyAddress().Hex() @@ -189,7 +188,7 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture executeVaultAllowListSecretsCreateTest(t, createEnc, secretID, owner, owner, gwURL, namespaces, sc, wfReg) t.Run("binary_encoded_shares", func(t *testing.T) { - executeVaultBinaryEncodedSharesSmokeTest(t, testEnv, secretID, "main", createValue, ulCh, bmCh) + executeVaultBinaryEncodedSharesSmokeTest(t, verifier, secretID, "main", createValue, ulCh, bmCh) }) executeVaultSecretsUpdateTest(t, updateEnc, secretID, owner, owner, gwURL, namespaces, sc, wfReg) executeVaultSecretsListTest(t, secretID, owner, owner, gwURL, "main", sc, wfReg) @@ -202,13 +201,15 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture {Name: "allowlist-main-not-found", SecretKey: secretID, SecretNamespace: "main", ExpectNotFound: true}, {Name: "allowlist-alt-updated", SecretKey: secretID, SecretNamespace: "alt", ExpectedValue: updateValue}, } - workflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "allowlist-lifecycle", []vaultWorkflowPhase{ - {Name: "allowlist-updated", Checks: updatedChecks}, - {Name: "allowlist-final-verify", Checks: finalChecks}, - }) - waitForVaultWorkflowPhase(t, workflowID, "allowlist-updated", ulCh, bmCh) + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "allowlist-updated", + Checks: updatedChecks, + }, ulCh, bmCh) executeVaultSecretsDeleteTest(t, secretID, owner, owner, gwURL, []string{"main"}, sc, wfReg) - waitForVaultWorkflowPhase(t, workflowID, "allowlist-final-verify", ulCh, bmCh) + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "allowlist-final-verify", + Checks: finalChecks, + }, ulCh, bmCh) executeVaultSecretsDeleteTest(t, secretID, owner, owner, gwURL, []string{"alt"}, sc, wfReg) }) @@ -235,15 +236,15 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture t_helpers.ShutdownChipSinkWithDrain(ctx, sink, ulCh, bmCh) }) executeVaultSecretsIdentifierValidationTest(t, enc, owner, gwURL, sc, wfReg) - executeVaultSecretsGetInvalidIdentifierViaWorkflowTest(t, subEnv, "vget1", ulCh, bmCh) - executeVaultSecretsGetBatchTooBigViaWorkflowTest(t, subEnv, "vget2", ulCh, bmCh) + executeVaultSecretsGetInvalidIdentifierViaWorkflowTest(t, verifier, ulCh, bmCh) + executeVaultSecretsGetBatchTooBigViaWorkflowTest(t, verifier, ulCh, bmCh) }) t.Run("pending_queue_blob_batching_many_concurrent_creates", func(t *testing.T) { ExecuteVaultBlobBatchingSmokeTest(t, fixture, testEnv) }) t.Run("include_invalid_pending_items_liveness", func(t *testing.T) { - ExecuteVaultIncludeInvalidLivenessSmokeTest(t, fixture, testEnv) + ExecuteVaultIncludeInvalidLivenessSmokeTest(t, fixture, testEnv, verifier) }) } @@ -289,6 +290,9 @@ func ExecuteVaultMixedAuthTest(t *testing.T, fixture *vaultScenarioFixture, test jwtAuth := newJWTVaultRequestAuth(issuer, orgID, derivedJWTWorkflowOwner, vaultParsedPublicKey, false) workflowOwnerAddress := common.HexToAddress(workflowOwner) + // Shared by all mixed-auth phase checks below. + verifier := deployVaultVerifierWorkflow(t, testEnv, fixture.TriggerAuth, "mixed-auth-verifier") + t.Run("jwt_crud_with_workflow_owner", func(t *testing.T) { secretID := uniqueVaultSecretID("jwt") createValue := "secret-jwt-workflow-owner" @@ -407,39 +411,34 @@ func ExecuteVaultMixedAuthTest(t *testing.T, fixture *vaultScenarioFixture, test executeVaultSecretsCreateWithAuth(t, allowlistAuth, allowlistCreateEnc, allowlistSecretID, workflowOwner, gwURL, []string{"main"}) executeVaultSecretsCreateWithAuth(t, jwtAuth, jwtCreateEnc, jwtSecretID, derivedJWTWorkflowOwner, gwURL, []string{"main"}) // Workflow phases validate only allow-listed reads (workflow runs as EOA). JWT-backed keys are exercised via Gateway SecretsList. - workflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "mixed-lifecycle", []vaultWorkflowPhase{ - { - Name: "mixed-created", - Checks: []vaultWorkflowCheck{ - {Name: "mixed-allowlist-create-get-main", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectedValue: allowlistCreateValue}, - }, - }, - { - Name: "mixed-updated", - Checks: []vaultWorkflowCheck{ - {Name: "mixed-allowlist-own-update-main", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectedValue: allowlistUpdateValue}, - }, + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "mixed-created", + Checks: []vaultWorkflowCheck{ + {Name: "mixed-allowlist-create-get-main", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectedValue: allowlistCreateValue}, }, - { - Name: "mixed-deleted", - Checks: []vaultWorkflowCheck{ - {Name: "mixed-allowlist-delete-not-found", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectNotFound: true}, - }, - }, - }) - waitForVaultWorkflowPhase(t, workflowID, "mixed-created", ulCh, bmCh) + }, ulCh, bmCh) executeVaultSecretsListWithAuth(t, jwtAuth, []string{jwtSecretID}, derivedJWTWorkflowOwner, gwURL, "main") executeVaultSecretsUpdateWithAuth(t, allowlistAuth, allowlistUpdateEnc, allowlistSecretID, workflowOwner, gwURL, []string{"main"}) executeVaultSecretsUpdateWithAuth(t, jwtAuth, jwtUpdateEnc, jwtSecretID, derivedJWTWorkflowOwner, gwURL, []string{"main"}) - waitForVaultWorkflowPhase(t, workflowID, "mixed-updated", ulCh, bmCh) + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "mixed-updated", + Checks: []vaultWorkflowCheck{ + {Name: "mixed-allowlist-own-update-main", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectedValue: allowlistUpdateValue}, + }, + }, ulCh, bmCh) executeVaultSecretsListWithAuth(t, allowlistAuth, []string{allowlistSecretID}, workflowOwner, gwURL, "main") executeVaultSecretsListWithAuth(t, jwtAuth, []string{jwtSecretID}, derivedJWTWorkflowOwner, gwURL, "main") executeVaultSecretsDeleteWithAuth(t, allowlistAuth, allowlistSecretID, workflowOwner, gwURL, []string{"main"}) executeVaultSecretsDeleteWithAuth(t, jwtAuth, jwtSecretID, derivedJWTWorkflowOwner, gwURL, []string{"main"}) - waitForVaultWorkflowPhase(t, workflowID, "mixed-deleted", ulCh, bmCh) + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "mixed-deleted", + Checks: []vaultWorkflowCheck{ + {Name: "mixed-allowlist-delete-not-found", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectNotFound: true}, + }, + }, ulCh, bmCh) executeVaultJWTSecretsListAbsentFromNamespace(t, issuer, vaultParsedPublicKey, jwtSecretID, orgID, derivedJWTWorkflowOwner, gwURL, "main") }) @@ -453,23 +452,18 @@ func ExecuteVaultMixedAuthTest(t *testing.T, fixture *vaultScenarioFixture, test require.NoError(t, err) executeVaultSecretsCreateWithAuth(t, allowlistAuth, createEncAllow, sharedKey, workflowOwner, gwURL, []string{"main"}) - // Vault workflow returns after the FIRST phase whose checks succeed (see vaultsecret/main.go). - // Two phases cannot share the same success predicate; otherwise the later phase never emits - // "Vault secret workflow phase completed: ..." and waits time out (4m). - crossWorkflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "mixed-cross-isolation", []vaultWorkflowPhase{ - { - Name: "cross-isolation-stable", - Checks: []vaultWorkflowCheck{ - {Name: "cross-share-key-allowlist-value", SecretKey: sharedKey, SecretNamespace: "main", ExpectedValue: allowlistValue}, - }, + crossPhase := vaultWorkflowPhase{ + Name: "cross-isolation-stable", + Checks: []vaultWorkflowCheck{ + {Name: "cross-share-key-allowlist-value", SecretKey: sharedKey, SecretNamespace: "main", ExpectedValue: allowlistValue}, }, - }) - waitForVaultWorkflowPhase(t, crossWorkflowID, "cross-isolation-stable", ulCh, bmCh) + } + triggerAndAwaitVaultWorkflowPhase(t, verifier, crossPhase, ulCh, bmCh) // JWT cannot overwrite the allow-listed row: identifiers + labels are partitioned by owner address. tryJWTSignedVaultSecretsUpdate(t, jwtAuth, jwtAuth.requestOwner, jwtCrossEnc, sharedKey, gwURL, []string{"main"}) - // Confirm on a subsequent cron tick that secrets still satisfy the same invariant. - waitForVaultWorkflowPhase(t, crossWorkflowID, "cross-isolation-stable", ulCh, bmCh) + // Confirm on a subsequent trigger that secrets still satisfy the same invariant. + triggerAndAwaitVaultWorkflowPhase(t, verifier, crossPhase, ulCh, bmCh) executeVaultSecretsDeleteWithAuth(t, allowlistAuth, sharedKey, workflowOwner, gwURL, []string{"main"}) // `sharedKey` only ever existed under the allowlisted workflow owner; JWT-targeted deletes use the @@ -499,7 +493,7 @@ func ExecuteVaultMixedAuthTest(t *testing.T, fixture *vaultScenarioFixture, test // ExecuteVaultIncludeInvalidLivenessSmokeTest verifies that an erroring workflow GetSecrets for a // deleted secret does not stall concurrent valid gateway creates while include-invalid is enabled. -func ExecuteVaultIncludeInvalidLivenessSmokeTest(t *testing.T, fixture *vaultScenarioFixture, testEnv *ttypes.TestEnvironment) { +func ExecuteVaultIncludeInvalidLivenessSmokeTest(t *testing.T, fixture *vaultScenarioFixture, testEnv *ttypes.TestEnvironment, verifier *vaultVerifierHandle) { t.Helper() testLogger := framework.L @@ -533,15 +527,6 @@ func ExecuteVaultIncludeInvalidLivenessSmokeTest(t *testing.T, fixture *vaultSce t_helpers.ShutdownChipSinkWithDrain(ctx, sink, ulCh, bmCh) }) - workflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "include-invalid-liveness", []vaultWorkflowPhase{ - { - Name: "deleted-secret-not-found", - Checks: []vaultWorkflowCheck{ - {Name: "get-deleted-not-found", SecretKey: deletedSecretID, SecretNamespace: "main", ExpectNotFound: true}, - }, - }, - }) - liveCreateRequestID := uuid.New().String() liveCreateRequest := newVaultJSONRequest(t, liveCreateRequestID, vaulttypes.MethodSecretsCreate, &vault_helpers.CreateSecretsRequest{ RequestId: liveCreateRequestID, @@ -556,11 +541,30 @@ func ExecuteVaultIncludeInvalidLivenessSmokeTest(t *testing.T, fixture *vaultSce createErrCh <- tryValidateVaultSecretsCreateResponse(gwURL, liveCreateRequest, liveCreateRequestID, liveSecretID, []string{owner}, namespaces) }() - waitForVaultWorkflowPhase(t, workflowID, "deleted-secret-not-found", ulCh, bmCh) + waitForDeletedSecretNotFoundPhase(t, verifier, deletedSecretID, ulCh, bmCh) <-createDone require.NoError(t, <-createErrCh) } +// waitForDeletedSecretNotFoundPhase asserts the verifier workflow sees +// deletedSecretID as not found. +func waitForDeletedSecretNotFoundPhase( + t *testing.T, + verifier *vaultVerifierHandle, + deletedSecretID string, + ulCh chan *workflowevents.UserLogs, + bmCh chan *commonevents.BaseMessage, +) { + t.Helper() + + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "deleted-secret-not-found", + Checks: []vaultWorkflowCheck{ + {Name: "get-deleted-not-found", SecretKey: deletedSecretID, SecretNamespace: "main", ExpectNotFound: true}, + }, + }, ulCh, bmCh) +} + // local pending queue and exercises batched pending-queue blobs (beyond the legacy per-blob single-item cap). func ExecuteVaultBlobBatchingSmokeTest(t *testing.T, fixture *vaultScenarioFixture, testEnv *ttypes.TestEnvironment) { t.Helper() @@ -989,28 +993,20 @@ func TestMustMintVaultJWTForRequest_UsesRawRequestDigest(t *testing.T) { } func executeVaultSecretsGetInvalidIdentifierViaWorkflowTest( - t *testing.T, testEnv *ttypes.TestEnvironment, - workflowBaseName string, + t *testing.T, verifier *vaultVerifierHandle, userLogsCh chan *workflowevents.UserLogs, baseMessageCh chan *commonevents.BaseMessage, ) { testLogger := framework.L testLogger.Info().Msg("Verifying get secret is rejected for invalid identifier via workflow...") - const workflowFileLocation = "./vaultsecret/main.go" - - workflowName := t_helpers.UniqueWorkflowName(testEnv, workflowBaseName) - workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &vaultsecret_config.Config{ + // Success is logged only after both the invalid key and invalid namespace are rejected. + triggerAndAwaitVaultVerifierLog(t, verifier, vaultsecret_config.TriggerInput{ + ExpectInvalidIdentifier: true, SecretKey: "invalid-key-with-hyphens", // hyphen not in [a-zA-Z0-9_]; tests invalid key SecretNamespace: "main", SecretKey2: "validkey", SecretNamespace2: "invalid-namespace-with-hyphens", // hyphen not in [a-zA-Z0-9_]; tests invalid namespace - ExpectInvalidIdentifier: true, - }, workflowFileLocation) - - // Both invalid-key and invalid-namespace checks run in the same cron trigger; a single - // success log is emitted only after both GetSecret calls are correctly rejected. - t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, - "Vault get correctly rejected invalid identifier", 4*time.Minute, t_helpers.WithUserLogWorkflowID(workflowID)) + }, "Vault get correctly rejected invalid identifier", userLogsCh, baseMessageCh) testLogger.Info().Msg("Vault get invalid identifier via workflow test completed") } @@ -1018,15 +1014,12 @@ func executeVaultSecretsGetInvalidIdentifierViaWorkflowTest( // GetSecrets batch is rejected with the real user-facing cause, asserting the // message survives the workflow-DON -> vault-DON remote capability hop. func executeVaultSecretsGetBatchTooBigViaWorkflowTest( - t *testing.T, testEnv *ttypes.TestEnvironment, - workflowBaseName string, + t *testing.T, verifier *vaultVerifierHandle, userLogsCh chan *workflowevents.UserLogs, baseMessageCh chan *commonevents.BaseMessage, ) { testLogger := framework.L testLogger.Info().Msg("Verifying get secret is rejected for oversized batch via workflow...") - const workflowFileLocation = "./vaultsecret/main.go" - // One over the limit; keys are valid identifiers so the batch size check // fires, not identifier validation. keys := make([]string, vaulttypes.MaxBatchSize+1) @@ -1034,15 +1027,11 @@ func executeVaultSecretsGetBatchTooBigViaWorkflowTest( keys[i] = fmt.Sprintf("batchkey%d", i) } - workflowName := t_helpers.UniqueWorkflowName(testEnv, workflowBaseName) - workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &vaultsecret_config.Config{ + triggerAndAwaitVaultVerifierLog(t, verifier, vaultsecret_config.TriggerInput{ + ExpectBatchTooBig: true, SecretNamespace: "main", BatchSecretKeys: keys, - ExpectBatchTooBig: true, - }, workflowFileLocation) - - t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, - "Vault get correctly rejected oversized batch", 4*time.Minute, t_helpers.WithUserLogWorkflowID(workflowID)) + }, "Vault get correctly rejected oversized batch", userLogsCh, baseMessageCh) testLogger.Info().Msg("Vault get oversized batch via workflow test completed") } diff --git a/system-tests/tests/smoke/cre/vault_don_test_helpers.go b/system-tests/tests/smoke/cre/vault_don_test_helpers.go index 3534f160941..9684f756860 100644 --- a/system-tests/tests/smoke/cre/vault_don_test_helpers.go +++ b/system-tests/tests/smoke/cre/vault_don_test_helpers.go @@ -3,6 +3,7 @@ package cre import ( "bytes" "context" + "crypto/ecdsa" "encoding/hex" "encoding/json" "errors" @@ -23,20 +24,21 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" - "github.com/smartcontractkit/tdh2/go/tdh2/tdh2easy" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" + "github.com/smartcontractkit/tdh2/go/tdh2/tdh2easy" vault_helpers "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault" capabilitiespb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" capabilities_registry_v2 "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/capabilities_registry_wrapper_v2" workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" + "github.com/smartcontractkit/chainlink-protos/cre/go/values" commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common" workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events" "github.com/smartcontractkit/chainlink-testing-framework/framework" @@ -48,12 +50,14 @@ import ( "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm" stvault "github.com/smartcontractkit/chainlink/system-tests/lib/cre/vault" creworkflow "github.com/smartcontractkit/chainlink/system-tests/lib/cre/workflow" + libcrypto "github.com/smartcontractkit/chainlink/system-tests/lib/crypto" vaultsecret_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/vaultsecret/config" t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" vaultjwt "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault" "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes" "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils" + "github.com/smartcontractkit/chainlink/v2/core/utils" ) const ( @@ -266,6 +270,23 @@ type vaultScenarioFixture struct { LinkingService *stvault.TestLinkingService GatewayURL *url.URL VaultPublicKey string + // TriggerAuth signs HTTP trigger requests to verifier workflows. + TriggerAuth *vaultTriggerAuth +} + +// vaultTriggerAuth is the HTTP trigger key pair: Address is the workflow's +// authorized key, PrivateKey signs the request JWT. +type vaultTriggerAuth struct { + Address common.Address + PrivateKey *ecdsa.PrivateKey +} + +func newVaultTriggerAuth() (*vaultTriggerAuth, error) { + addr, privateKey, err := libcrypto.GenerateNewKeyPair() + if err != nil { + return nil, err + } + return &vaultTriggerAuth{Address: addr, PrivateKey: privateKey}, nil } type vaultWorkflowCheck struct { @@ -324,6 +345,9 @@ func setupVaultScenarioFixture(t *testing.T, baseConfig *ttypes.TestConfig, useP linkingService, err := stvault.EnsureSharedTestLinkingServiceStarted() require.NoError(t, err) + triggerAuth, err := newVaultTriggerAuth() + require.NoError(t, err, "failed to generate vault verifier trigger signing key") + var testEnv *ttypes.TestEnvironment if usePerTestKeys { testEnv = t_helpers.SetupTestEnvironmentWithPerTestKeys(t, baseConfig) @@ -342,6 +366,7 @@ func setupVaultScenarioFixture(t *testing.T, baseConfig *ttypes.TestConfig, useP LinkingService: linkingService, GatewayURL: gatewayURL, VaultPublicKey: vaultPublicKey, + TriggerAuth: triggerAuth, } } @@ -1314,63 +1339,200 @@ func executeVaultJWTSecretsCreateUnauthorizedWithExtraClaimsTest( require.Contains(t, jsonResponse.Error.Error(), expectedAuthError) } -func startVaultSecretsWorkflowPhasesTest( - t *testing.T, testEnv *ttypes.TestEnvironment, - workflowBaseName string, - phases []vaultWorkflowPhase, -) string { +// vaultVerifierHandle is a deployed verifier workflow and what's needed to +// trigger it via the gateway. +type vaultVerifierHandle struct { + WorkflowName string + WorkflowID string + WorkflowOwner string + GatewayURL string + TriggerAuth *vaultTriggerAuth +} + +const ( + vaultVerifierTriggerRetryInterval = 2 * time.Second + vaultVerifierTriggerAttemptWindow = 30 * time.Second + vaultVerifierTriggerDeadline = 3 * time.Minute +) + +// deployVaultVerifierWorkflow deploys the vaultsecret workflow. Checks are +// passed per trigger, so one deployment serves every scenario. +func deployVaultVerifierWorkflow(t *testing.T, testEnv *ttypes.TestEnvironment, triggerAuth *vaultTriggerAuth, workflowBaseName string) *vaultVerifierHandle { t.Helper() testLogger := framework.L - testLogger.Info(). - Str("workflow_base_name", workflowBaseName). - Int("phase_count", len(phases)). - Msg("Starting vault workflow phase verification") - workflowName := t_helpers.UniqueWorkflowName(testEnv, workflowBaseName) - cfgPhases := make([]vaultsecret_config.Phase, 0, len(phases)) - for _, phase := range phases { - cfgChecks := make([]vaultsecret_config.Check, 0, len(phase.Checks)) - for _, check := range phase.Checks { - cfgChecks = append(cfgChecks, vaultsecret_config.Check{ - Name: check.Name, - SecretKey: check.SecretKey, - SecretNamespace: check.SecretNamespace, - ExpectedValue: check.ExpectedValue, - ExpectNotFound: check.ExpectNotFound, - }) - } - cfgPhases = append(cfgPhases, vaultsecret_config.Phase{ - Name: phase.Name, - Checks: cfgChecks, - }) + cfg := &vaultsecret_config.Config{AuthorizedKey: triggerAuth.Address.Hex()} + const workflowFileLocation = "./vaultsecret/main.go" + workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, cfg, workflowFileLocation) + + require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0]) + sc := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient + gatewayURL := mustVaultGatewayURL(t, testEnv) + return &vaultVerifierHandle{ + WorkflowName: workflowName, + WorkflowID: workflowID, + WorkflowOwner: strings.ToLower(sc.MustGetRootKeyAddress().Hex()), + GatewayURL: gatewayURL.String(), + TriggerAuth: triggerAuth, } +} - cfg := &vaultsecret_config.Config{Phases: cfgPhases} - const workflowFileLocation = "./vaultsecret/main.go" - return t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, cfg, workflowFileLocation) +// sendVaultVerifierTriggerRequest sends one signed HTTP trigger request. It +// returns a retryable error while the workflow is still loading or if the +// gateway rejects the request. +func sendVaultVerifierTriggerRequest(t *testing.T, handle *vaultVerifierHandle, input vaultsecret_config.TriggerInput) error { + t.Helper() + + inputJSON, err := json.Marshal(input) + require.NoError(t, err, "failed to marshal vault verifier trigger input") + + triggerPayload := gateway_common.HTTPTriggerRequest{ + Workflow: gateway_common.WorkflowSelector{ + WorkflowOwner: handle.WorkflowOwner, + WorkflowName: handle.WorkflowName, + WorkflowTag: creworkflow.DefaultWorkflowTag, + WorkflowID: handle.WorkflowID, + }, + Input: json.RawMessage(inputJSON), + } + params, err := json.Marshal(triggerPayload) + require.NoError(t, err, "failed to marshal vault verifier trigger request") + rawParams := json.RawMessage(params) + + // Fresh ID per attempt: duplicate IDs get a cached response without re-executing. + req := jsonrpc.Request[json.RawMessage]{ + Version: jsonrpc.JsonRpcVersion, + ID: uuid.New().String(), + Method: gateway_common.MethodWorkflowExecute, + Params: &rawParams, + } + + token, err := utils.CreateRequestJWT(req) + require.NoError(t, err, "failed to create vault verifier trigger JWT") + tokenString, err := token.SignedString(handle.TriggerAuth.PrivateKey) + require.NoError(t, err, "failed to sign vault verifier trigger JWT") + req.Auth = tokenString + + requestBody, err := json.Marshal(req) + require.NoError(t, err, "failed to marshal vault verifier trigger request body") + + httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handle.GatewayURL, bytes.NewBuffer(requestBody)) + require.NoError(t, err, "failed to create vault verifier trigger HTTP request") + httpReq.Header.Set("Content-Type", "application/jsonrpc") + httpReq.Header.Set("Accept", "application/json") + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute vault verifier trigger request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read vault verifier trigger response body: %w", err) + } + framework.L.Info(). + Str("workflow", handle.WorkflowName). + Int("status", resp.StatusCode). + Msgf("Vault verifier trigger response: %s", string(body)) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("vault verifier trigger request rejected with status %d: %s", resp.StatusCode, string(body)) + } + + var parsed jsonrpc.Response[json.RawMessage] + if err := json.Unmarshal(body, &parsed); err != nil { + return fmt.Errorf("failed to unmarshal vault verifier trigger response: %w", err) + } + if parsed.Error != nil { + return fmt.Errorf("vault verifier trigger request returned error: %s", parsed.Error.Error()) + } + if parsed.ID != req.ID { + return fmt.Errorf("vault verifier trigger response ID mismatch: got %q want %q", parsed.ID, req.ID) + } + return nil } -func waitForVaultWorkflowPhase( +// triggerAndAwaitVaultVerifierLog triggers the verifier and waits for +// expectedUserLog, re-triggering until the workflow is loaded and checks pass. +func triggerAndAwaitVaultVerifierLog( t *testing.T, - workflowID, phaseName string, + handle *vaultVerifierHandle, + input vaultsecret_config.TriggerInput, + expectedUserLog string, userLogsCh chan *workflowevents.UserLogs, baseMessageCh chan *commonevents.BaseMessage, ) { t.Helper() testLogger := framework.L - t_helpers.WatchWorkflowLogs( - t, - testLogger, - userLogsCh, - baseMessageCh, - t_helpers.WorkflowEngineInitErrorLog, - "Vault secret workflow phase completed: "+phaseName, - 1*time.Minute, - t_helpers.WithUserLogWorkflowID(workflowID), - ) - testLogger.Info().Str("phase_name", phaseName).Msg("Vault secret workflow phase completed") + + ctx, cancelFn := context.WithTimeoutCause(t.Context(), vaultVerifierTriggerDeadline, errors.New("timed out waiting for vault verifier workflow log")) + defer cancelFn() + cancelCtx, cancelCauseFn := context.WithCancelCause(ctx) + defer cancelCauseFn(nil) + + // Fail fast on engine init errors, as WatchWorkflowLogs does. + go func() { + t_helpers.FailOnBaseMessage(cancelCtx, cancelCauseFn, t, testLogger, baseMessageCh, + t_helpers.WorkflowEngineInitErrorLog, t_helpers.WithBaseMessageWorkflowID(handle.WorkflowID)) + }() + + var lastErr error + for { + if err := sendVaultVerifierTriggerRequest(t, handle, input); err != nil { + lastErr = err + testLogger.Warn().Str("workflow", handle.WorkflowName).Err(err).Msg("Vault verifier trigger not accepted yet; will retry") + } else { + attemptCtx, attemptCancel := context.WithTimeoutCause(cancelCtx, vaultVerifierTriggerAttemptWindow, errors.New("vault verifier trigger attempt window expired")) + _, waitErr := t_helpers.WaitForUserLog(attemptCtx, testLogger, userLogsCh, expectedUserLog, t_helpers.WithUserLogWorkflowID(handle.WorkflowID)) + attemptCancel() + if waitErr == nil { + testLogger.Info().Str("workflow", handle.WorkflowName).Str("expected_log", expectedUserLog).Msg("Vault verifier workflow log observed") + return + } + lastErr = waitErr + testLogger.Warn().Str("workflow", handle.WorkflowName).Err(waitErr).Msg("Vault verifier phase not yet satisfied; will re-trigger") + } + + select { + case <-cancelCtx.Done(): + require.FailNow(t, "timed out waiting for vault verifier workflow log", + "workflow=%s expectedLog=%q lastErr=%v", handle.WorkflowName, expectedUserLog, lastErr) + return + case <-time.After(vaultVerifierTriggerRetryInterval): + } + } +} + +// triggerAndAwaitVaultWorkflowPhase runs a single phase and waits for its completion log. +func triggerAndAwaitVaultWorkflowPhase( + t *testing.T, + handle *vaultVerifierHandle, + phase vaultWorkflowPhase, + userLogsCh chan *workflowevents.UserLogs, + baseMessageCh chan *commonevents.BaseMessage, +) { + t.Helper() + + input := vaultsecret_config.TriggerInput{Phases: []vaultsecret_config.Phase{vaultWorkflowPhaseToConfig(phase)}} + triggerAndAwaitVaultVerifierLog(t, handle, input, "Vault secret workflow phase completed: "+phase.Name, userLogsCh, baseMessageCh) +} + +func vaultWorkflowPhaseToConfig(phase vaultWorkflowPhase) vaultsecret_config.Phase { + checks := make([]vaultsecret_config.Check, 0, len(phase.Checks)) + for _, check := range phase.Checks { + checks = append(checks, vaultsecret_config.Check{ + Name: check.Name, + SecretKey: check.SecretKey, + SecretNamespace: check.SecretNamespace, + ExpectedValue: check.ExpectedValue, + ExpectNotFound: check.ExpectNotFound, + }) + } + return vaultsecret_config.Phase{Name: phase.Name, Checks: checks} } func executeVaultSecretsUpdateTest(t *testing.T, encryptedSecret, secretID, requestOwner, expectedResponseOwner, gatewayURL string, namespaces []string, sethClient *seth.Client, wfRegistryContract *workflow_registry_v2_wrapper.WorkflowRegistry) { @@ -1394,7 +1556,7 @@ func executeVaultSecretsDeleteTest(t *testing.T, secretID, requestOwner, expecte // workflow triggers a capability get. func executeVaultBinaryEncodedSharesSmokeTest( t *testing.T, - testEnv *ttypes.TestEnvironment, + handle *vaultVerifierHandle, secretID, namespace, expectedPlaintext string, userLogsCh chan *workflowevents.UserLogs, baseMessageCh chan *commonevents.BaseMessage, @@ -1403,7 +1565,7 @@ func executeVaultBinaryEncodedSharesSmokeTest( framework.L.Info().Msg("Verifying workflow secret fetch and binary share encoding when optimizations are enabled...") - workflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "binary-shares", []vaultWorkflowPhase{{ + phase := vaultWorkflowPhase{ Name: "binary-shares-fetch", Checks: []vaultWorkflowCheck{{ Name: "binary-shares-main", @@ -1411,8 +1573,8 @@ func executeVaultBinaryEncodedSharesSmokeTest( SecretNamespace: namespace, ExpectedValue: expectedPlaintext, }}, - }}) - waitForVaultWorkflowPhase(t, workflowID, "binary-shares-fetch", userLogsCh, baseMessageCh) + } + triggerAndAwaitVaultWorkflowPhase(t, handle, phase, userLogsCh, baseMessageCh) } // updateVaultCapabilityConfigInRegistry updates the on-chain capabilities registry @@ -1503,8 +1665,8 @@ func updateVaultCapabilityConfigInRegistry(t *testing.T, testEnv *ttypes.TestEnv _, err = deployerClient.Decode(capReg.UpdateDONByName(deployerClient.NewTXOpts(), don.Name, updateParams)) require.NoError(t, err, "UpdateDONByName tx failed") - testLogger.Info().Msg("Waiting for registry syncer to propagate the on-chain config change...") - time.Sleep(15 * time.Second) // registry syncer polls every 12s; one tick + margin + // No wait: nodes pick up the config on their next registry syncer tick (12s). + testLogger.Info().Msg("Updated vault capability config in capabilities registry") } func allowlistRequest(t *testing.T, owner string, request jsonrpc.Request[json.RawMessage], sethClient *seth.Client, wfRegistryContract *workflow_registry_v2_wrapper.WorkflowRegistry) { diff --git a/system-tests/tests/smoke/cre/vaultsecret/config/config.go b/system-tests/tests/smoke/cre/vaultsecret/config/config.go index f6f61d8b7a5..c8177b5b184 100644 --- a/system-tests/tests/smoke/cre/vaultsecret/config/config.go +++ b/system-tests/tests/smoke/cre/vaultsecret/config/config.go @@ -1,63 +1,33 @@ package config +// Check is a single secret assertion evaluated by the workflow. type Check struct { - Name string `yaml:"name,omitempty"` - SecretKey string `yaml:"secretKey"` - SecretNamespace string `yaml:"secretNamespace"` - ExpectedValue string `yaml:"expectedValue,omitempty"` - ExpectNotFound bool `yaml:"expectNotFound"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + SecretKey string `yaml:"secretKey" json:"secretKey"` + SecretNamespace string `yaml:"secretNamespace" json:"secretNamespace"` + ExpectedValue string `yaml:"expectedValue,omitempty" json:"expectedValue,omitempty"` + ExpectNotFound bool `yaml:"expectNotFound" json:"expectNotFound"` } type Phase struct { - Name string `yaml:"name"` - Checks []Check `yaml:"checks"` + Name string `yaml:"name" json:"name"` + Checks []Check `yaml:"checks" json:"checks"` } type Config struct { - Phases []Phase `yaml:"phases"` - Checks []Check `yaml:"checks"` - - // Legacy single-check fields kept for compatibility with any older callers. - SecretKey string `yaml:"secretKey,omitempty"` - SecretNamespace string `yaml:"secretNamespace,omitempty"` - SecretKey2 string `yaml:"secretKey2"` - SecretNamespace2 string `yaml:"secretNamespace2"` - ExpectedValue string `yaml:"expectedValue,omitempty"` - ExpectNotFound bool `yaml:"expectNotFound,omitempty"` - ExpectInvalidIdentifier bool `yaml:"expectInvalidIdentifier"` - ExpectBatchTooBig bool `yaml:"expectBatchTooBig,omitempty"` - BatchSecretKeys []string `yaml:"batchSecretKeys,omitempty"` + // AuthorizedKey is the EVM address allowed to sign HTTP trigger requests. + AuthorizedKey string `yaml:"authorizedKey" json:"authorizedKey"` } -func (c Config) EffectiveChecks() []Check { - if len(c.Checks) > 0 { - return c.Checks - } - - if c.SecretKey == "" && c.SecretNamespace == "" { - return nil - } - - return []Check{{ - SecretKey: c.SecretKey, - SecretNamespace: c.SecretNamespace, - ExpectedValue: c.ExpectedValue, - ExpectNotFound: c.ExpectNotFound, - }} -} - -func (c Config) EffectivePhases() []Phase { - if len(c.Phases) > 0 { - return c.Phases - } - - checks := c.EffectiveChecks() - if len(checks) == 0 { - return nil - } - - return []Phase{{ - Name: "default", - Checks: checks, - }} +type TriggerInput struct { + // Phases are evaluated in order; the first whose checks all pass logs completion. + Phases []Phase `json:"phases,omitempty"` + + ExpectInvalidIdentifier bool `json:"expectInvalidIdentifier,omitempty"` + SecretKey string `json:"secretKey,omitempty"` + SecretNamespace string `json:"secretNamespace,omitempty"` + SecretKey2 string `json:"secretKey2,omitempty"` + SecretNamespace2 string `json:"secretNamespace2,omitempty"` + ExpectBatchTooBig bool `json:"expectBatchTooBig,omitempty"` + BatchSecretKeys []string `json:"batchSecretKeys,omitempty"` } diff --git a/system-tests/tests/smoke/cre/vaultsecret/go.mod b/system-tests/tests/smoke/cre/vaultsecret/go.mod index 936986b55dd..c03e186aca2 100644 --- a/system-tests/tests/smoke/cre/vaultsecret/go.mod +++ b/system-tests/tests/smoke/cre/vaultsecret/go.mod @@ -4,7 +4,7 @@ go 1.26.5 require ( github.com/smartcontractkit/cre-sdk-go v1.16.0 - github.com/smartcontractkit/cre-sdk-go/capabilities/scheduler/cron v1.3.0 + github.com/smartcontractkit/cre-sdk-go/capabilities/networking/http v1.4.1-0.20260617164717-4ae5e15fa5ac gopkg.in/yaml.v3 v3.0.1 ) diff --git a/system-tests/tests/smoke/cre/vaultsecret/go.sum b/system-tests/tests/smoke/cre/vaultsecret/go.sum index 2d4e1afc506..42d865f6eed 100644 --- a/system-tests/tests/smoke/cre/vaultsecret/go.sum +++ b/system-tests/tests/smoke/cre/vaultsecret/go.sum @@ -28,8 +28,8 @@ github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260707203317-661b54 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260707203317-661b54b51a33/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= github.com/smartcontractkit/cre-sdk-go v1.16.0 h1:xFdLz8nf4pEzMkS+4Qq43aMvfl4EcnSshhsssM5ToIg= github.com/smartcontractkit/cre-sdk-go v1.16.0/go.mod h1:hhPdldSAv6CYSeHZzGOqFKDPe5zA6fuiBQpC0Ue/HI0= -github.com/smartcontractkit/cre-sdk-go/capabilities/scheduler/cron v1.3.0 h1:qBZ4y6qlTOynSpU1QAi2Fgr3tUZQ332b6hit9EVZqkk= -github.com/smartcontractkit/cre-sdk-go/capabilities/scheduler/cron v1.3.0/go.mod h1:Rzhy75vD3FqQo/SV6lypnxIwjWac6IOWzI5BYj3tYMU= +github.com/smartcontractkit/cre-sdk-go/capabilities/networking/http v1.4.1-0.20260617164717-4ae5e15fa5ac h1:t9WmKs/fjV+FiQmfGOCZzbQWorHVCcEQ4DRXBmczxw0= +github.com/smartcontractkit/cre-sdk-go/capabilities/networking/http v1.4.1-0.20260617164717-4ae5e15fa5ac/go.mod h1:GrSp5EegpvJNfQkBCN1klmrMBwaVTRAlvt020Yx0JJ4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= diff --git a/system-tests/tests/smoke/cre/vaultsecret/main.go b/system-tests/tests/smoke/cre/vaultsecret/main.go index 2382e9dd4b4..8c86e61e823 100644 --- a/system-tests/tests/smoke/cre/vaultsecret/main.go +++ b/system-tests/tests/smoke/cre/vaultsecret/main.go @@ -3,13 +3,14 @@ package main import ( + "encoding/json" "fmt" "log/slog" "strings" "gopkg.in/yaml.v3" - "github.com/smartcontractkit/cre-sdk-go/capabilities/scheduler/cron" + httptrigger "github.com/smartcontractkit/cre-sdk-go/capabilities/networking/http" "github.com/smartcontractkit/cre-sdk-go/cre" "github.com/smartcontractkit/cre-sdk-go/cre/wasm" @@ -26,25 +27,39 @@ func main() { }).Run(RunVaultSecretWorkflow) } +// RunVaultSecretWorkflow is an HTTP-triggered verifier; checks arrive per +// invocation in config.TriggerInput. func RunVaultSecretWorkflow(cfg config.Config, _ *slog.Logger, _ cre.SecretsProvider) (cre.Workflow[config.Config], error) { return cre.Workflow[config.Config]{ cre.Handler( - // cron-trigger@1.0.0 rejects schedules faster than 30s ("maximum fastest cron schedule is 30s"). - cron.Trigger(&cron.Config{Schedule: "*/30 * * * * *"}), + httptrigger.Trigger(&httptrigger.Config{ + AuthorizedKeys: []*httptrigger.AuthorizedKey{ + { + Type: httptrigger.KeyType_KEY_TYPE_ECDSA_EVM, + PublicKey: cfg.AuthorizedKey, + }, + }, + }), onTrigger, ), }, nil } -func onTrigger(cfg config.Config, runtime cre.Runtime, _ *cron.Payload) (string, error) { - if cfg.ExpectInvalidIdentifier { - return evaluateInvalidIdentifiers(cfg, runtime) +func onTrigger(cfg config.Config, runtime cre.Runtime, trigger *httptrigger.Payload) (string, error) { + var input config.TriggerInput + if len(trigger.Input) > 0 { + if err := json.Unmarshal(trigger.Input, &input); err != nil { + return "", fmt.Errorf("failed to unmarshal trigger input: %w", err) + } + } + if input.ExpectInvalidIdentifier { + return evaluateInvalidIdentifiers(input, runtime) } - if cfg.ExpectBatchTooBig { - return evaluateBatchTooBig(cfg, runtime) + if input.ExpectBatchTooBig { + return evaluateBatchTooBig(input, runtime) } - phases := cfg.EffectivePhases() + phases := input.Phases if len(phases) == 0 { return "", fmt.Errorf("no vault workflow phases configured") } @@ -73,25 +88,25 @@ func onTrigger(cfg config.Config, runtime cre.Runtime, _ *cron.Payload) (string, return "", fmt.Errorf("no vault workflow phase matched current state: %w", lastErr) } -func evaluateInvalidIdentifiers(cfg config.Config, runtime cre.Runtime) (string, error) { +func evaluateInvalidIdentifiers(input config.TriggerInput, runtime cre.Runtime) (string, error) { _, err := runtime.GetSecret(&cre.SecretRequest{ - Namespace: cfg.SecretNamespace, - Id: cfg.SecretKey, + Namespace: input.SecretNamespace, + Id: input.SecretKey, }).Await() if err == nil { - runtime.Logger().Error("Expected identifier validation to fail but GetSecret succeeded", "secretKey", cfg.SecretKey) - return "", fmt.Errorf("expected identifier validation failure for key=%s, but secret was retrieved", cfg.SecretKey) + runtime.Logger().Error("Expected identifier validation to fail but GetSecret succeeded", "secretKey", input.SecretKey) + return "", fmt.Errorf("expected identifier validation failure for key=%s, but secret was retrieved", input.SecretKey) } - runtime.Logger().Info("Vault get correctly rejected invalid identifier", "secretKey", cfg.SecretKey, "error", err) + runtime.Logger().Info("Vault get correctly rejected invalid identifier", "secretKey", input.SecretKey, "error", err) - if cfg.SecretKey2 != "" || cfg.SecretNamespace2 != "" { - key2 := cfg.SecretKey2 + if input.SecretKey2 != "" || input.SecretNamespace2 != "" { + key2 := input.SecretKey2 if key2 == "" { - key2 = cfg.SecretKey + key2 = input.SecretKey } - ns2 := cfg.SecretNamespace2 + ns2 := input.SecretNamespace2 if ns2 == "" { - ns2 = cfg.SecretNamespace + ns2 = input.SecretNamespace } _, err2 := runtime.GetSecret(&cre.SecretRequest{ Namespace: ns2, @@ -105,21 +120,21 @@ func evaluateInvalidIdentifiers(cfg config.Config, runtime cre.Runtime) (string, runtime.Logger().Info("Vault get correctly rejected invalid identifier", "secretKey", key2, "error", err2) } - return fmt.Sprintf("Invalid identifier correctly rejected: key=%s", cfg.SecretKey), nil + return fmt.Sprintf("Invalid identifier correctly rejected: key=%s", input.SecretKey), nil } // evaluateBatchTooBig submits a GetSecrets batch larger than the vault request // batch size limit and verifies the rejection carries the real user-facing // cause across the workflow-DON -> vault-DON remote capability hop. -func evaluateBatchTooBig(cfg config.Config, runtime cre.Runtime) (string, error) { - if len(cfg.BatchSecretKeys) == 0 { +func evaluateBatchTooBig(input config.TriggerInput, runtime cre.Runtime) (string, error) { + if len(input.BatchSecretKeys) == 0 { return "", fmt.Errorf("expectBatchTooBig requires batchSecretKeys to be set") } - reqs := make([]*cre.SecretRequest, len(cfg.BatchSecretKeys)) - for i, key := range cfg.BatchSecretKeys { + reqs := make([]*cre.SecretRequest, len(input.BatchSecretKeys)) + for i, key := range input.BatchSecretKeys { reqs[i] = &cre.SecretRequest{ - Namespace: cfg.SecretNamespace, + Namespace: input.SecretNamespace, Id: key, } }