diff --git a/.changeset/flat-worms-teach.md b/.changeset/flat-worms-teach.md new file mode 100644 index 00000000000..a982cf507e0 --- /dev/null +++ b/.changeset/flat-worms-teach.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +Added feature to the EVM BalanceMonitor to allow tracking balance of ERC-20 type fee tokens #added diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml index 89daed46fc1..1a59dd0ff94 100644 --- a/.github/workflows/cre-system-tests.yaml +++ b/.github/workflows/cre-system-tests.yaml @@ -190,7 +190,7 @@ jobs: # http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments name: integration deployment: false - timeout-minutes: 13 + timeout-minutes: 10 env: ENABLE_AUTO_QUARANTINE: "true" BILLING_PLATFORM_SERVICE_IMAGE: @@ -415,7 +415,7 @@ jobs: continue-on-error: ${{ env.ENABLE_AUTO_QUARANTINE == 'true' }} env: TEST_NAME: ${{ matrix.tests.test_name }} - TEST_TIMEOUT: 10m # let's leave 3 minutes for other steps (the whole job times out after 13 minutes) + TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes) RUN_QUARANTINED_TESTS: "true" # always run quarantined tests in CI TOPOLOGY_NAME: ${{ matrix.tests.topology }} CONFIDENTIAL_COMPUTE_ROOT: ${{ github.workspace }}/chainlink-confidential-compute diff --git a/core/capabilities/launcher.go b/core/capabilities/launcher.go index 24bf8c5adb7..0e99a1912f8 100644 --- a/core/capabilities/launcher.go +++ b/core/capabilities/launcher.go @@ -75,15 +75,19 @@ type launcher struct { // For V2 capabilities, shims are created once and their config is updated dynamically. type cachedShims struct { - combinedClients map[string]remote.CombinedClient - triggerSubscribers map[string]remote.TriggerSubscriber - triggerPublishers map[string]remote.TriggerPublisher - executableClients map[string]executable.Client - executableServers map[string]executable.Server + combinedClients map[shimID]remote.CombinedClient + triggerSubscribers map[shimID]remote.TriggerSubscriber + triggerPublishers map[shimID]remote.TriggerPublisher + executableClients map[shimID]executable.Client + executableServers map[shimID]executable.Server } -func shimKey(capID string, donID uint32, method string) string { - return fmt.Sprintf("%s:%d:%s", capID, donID, method) +// shimID identifies a single per-method remote shim. combinedClients are keyed with method == "", +// since a CombinedClient covers every method of a capability. +type shimID struct { + capID string + donID uint32 + method string } // TODO: add metric handler and instrument all the internal log.Error calls @@ -134,11 +138,11 @@ func NewLauncher( lggr: logger.Sugared(lggr).Named("CapabilitiesLauncher"), dispatcher: dispatcher, cachedShims: cachedShims{ - combinedClients: make(map[string]remote.CombinedClient), - triggerSubscribers: make(map[string]remote.TriggerSubscriber), - triggerPublishers: make(map[string]remote.TriggerPublisher), - executableClients: make(map[string]executable.Client), - executableServers: make(map[string]executable.Server), + combinedClients: make(map[shimID]remote.CombinedClient), + triggerSubscribers: make(map[shimID]remote.TriggerSubscriber), + triggerPublishers: make(map[shimID]remote.TriggerPublisher), + executableClients: make(map[shimID]executable.Client), + executableServers: make(map[shimID]executable.Server), }, registry: registry, workflowDonNotifier: workflowDonNotifier, @@ -345,6 +349,22 @@ func (w *launcher) onNewRegistry(ctx context.Context, metadataRegistry *registry w.lggr.Warn("My node doesn't belong to any DON families. No filtering will be applied.") } + belongsToAWorkflowDON := len(myWorkflowDONs) > 0 + belongsToACapabilityDON := len(myCapabilityDONs) > 0 + + // Prune shims for capabilities/methods/DONs that are no longer wanted, before the create/update + // loops below run. This must happen first: if a capability moved to a different DON ID, its old + // CombinedClient must be removed from the registry before the replacement is added under the new + // DON ID (registry.Add errors if an entry for the same capability ID is still present). + var wantedClientDONs, wantedServerDONs []registry.DON + if belongsToAWorkflowDON { + wantedClientDONs = remoteCapabilityDONs + } + if belongsToACapabilityDON { + wantedServerDONs = myCapabilityDONs + } + w.pruneStaleShims(ctx, computeWantedShimKeys(wantedClientDONs, wantedServerDONs)) + // Reconcile local capabilities: start/stop/restart capabilities based on registry state. if w.localCapMgr != nil { myDONs := make([]registry.DON, 0, len(myCapabilityDONs)+len(myWorkflowDONs)) @@ -355,7 +375,6 @@ func (w *launcher) onNewRegistry(ctx context.Context, metadataRegistry *registry } } - belongsToAWorkflowDON := len(myWorkflowDONs) > 0 if belongsToAWorkflowDON { myDON := myWorkflowDONs[0] @@ -373,7 +392,6 @@ func (w *launcher) onNewRegistry(ctx context.Context, metadataRegistry *registry } } - belongsToACapabilityDON := len(myCapabilityDONs) > 0 if belongsToACapabilityDON { for _, myDON := range myCapabilityDONs { w.serveCapabilities(ctx, w.myPeerID, myDON, remoteWorkflowDONs) @@ -547,9 +565,9 @@ func (w *launcher) addRemoteCapabilityV2(ctx context.Context, capID string, meth continue } - shimKey := shimKey(capID, remoteDON.ID, method) + key := shimID{capID: capID, donID: remoteDON.ID, method: method} if config.RemoteTriggerConfig != nil { // trigger - sub, alreadyExists := w.cachedShims.triggerSubscribers[shimKey] + sub, alreadyExists := w.cachedShims.triggerSubscribers[key] if !alreadyExists { sub = remote.NewTriggerSubscriber(capID, method, w.dispatcher, w.lggr) cc.SetTriggerSubscriber(method, sub) @@ -567,11 +585,11 @@ func (w *launcher) addRemoteCapabilityV2(ctx context.Context, capID string, meth w.lggr.Errorw("failed to start receiver", "capID", capID, "method", method, "error", err2) continue } - w.cachedShims.triggerSubscribers[shimKey] = sub + w.cachedShims.triggerSubscribers[key] = sub w.lggr.Infow("added new remote trigger subscriber", "capID", capID, "method", method) } } else { // executable - client, alreadyExists := w.cachedShims.executableClients[shimKey] + client, alreadyExists := w.cachedShims.executableClients[key] if !alreadyExists { client = executable.NewClient(info.ID, method, w.dispatcher, w.lggr) cc.SetExecutableClient(method, client) @@ -594,7 +612,7 @@ func (w *launcher) addRemoteCapabilityV2(ctx context.Context, capID string, meth w.lggr.Errorw("failed to start receiver", "capID", capID, "method", method, "error", err2) continue } - w.cachedShims.executableClients[shimKey] = client + w.cachedShims.executableClients[key] = client w.lggr.Infow("added new remote executable client", "capID", capID, "method", method) } } @@ -644,8 +662,8 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo if !ok { return fmt.Errorf("capability %s does not implement TriggerCapability", capID) } - shimKey := shimKey(capID, myDON.ID, method) - publisher, alreadyExists := w.cachedShims.triggerPublishers[shimKey] + key := shimID{capID: capID, donID: myDON.ID, method: method} + publisher, alreadyExists := w.cachedShims.triggerPublishers[key] if !alreadyExists { publisher = remote.NewTriggerPublisher( capID, @@ -665,7 +683,7 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo w.lggr.Errorw("failed to start receiver", "capID", capID, "method", method, "error", err2) continue } - w.cachedShims.triggerPublishers[shimKey] = publisher + w.cachedShims.triggerPublishers[key] = publisher w.lggr.Infow("added new remote trigger publisher", "capID", capID, "method", method) } } else { // executable @@ -674,8 +692,8 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo return fmt.Errorf("capability %s does not implement ExecutableCapability", capID) } - shimKey := shimKey(capID, myDON.ID, method) - server, alreadyExists := w.cachedShims.executableServers[shimKey] + key := shimID{capID: capID, donID: myDON.ID, method: method} + server, alreadyExists := w.cachedShims.executableServers[key] if !alreadyExists { server = executable.NewServer( info.ID, @@ -717,7 +735,7 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo w.lggr.Errorw("failed to start receiver", "capID", capID, "method", method, "error", err2) continue } - w.cachedShims.executableServers[shimKey] = server + w.cachedShims.executableServers[key] = server w.lggr.Infow("added new remote executable server", "capID", capID, "method", method) } } @@ -727,12 +745,149 @@ func (w *launcher) serveCapabilityV2(ctx context.Context, capID string, methodCo // retrieve or create a CombinedClient for the given capability func (w *launcher) getCombinedClient(info capabilities.CapabilityInfo) (remote.CombinedClient, bool) { - key := shimKey(info.ID, info.DON.ID, "") // empty method name - CombinedClient covers all methods + key := shimID{capID: info.ID, donID: info.DON.ID} // empty method name - CombinedClient covers all methods cc, exists := w.cachedShims.combinedClients[key] if !exists { // create a new combined client and cache it cc = remote.NewCombinedClient(info) w.cachedShims.combinedClients[key] = cc return cc, true } + // DON ID is unchanged (same key), but other DON metadata (members, families, name, IsPublic) + // may have changed - refresh it so the CombinedClient's Info() doesn't go stale. + cc.SetInfo(info) return cc, false } + +// wantedShimKeys is the set of shim keys expected to exist after a reconcile pass. +type wantedShimKeys struct { + combinedClients map[shimID]struct{} + triggerSubscribers map[shimID]struct{} + executableClients map[shimID]struct{} + triggerPublishers map[shimID]struct{} + executableServers map[shimID]struct{} +} + +// computeWantedShimKeys determines every shim key this reconcile pass will create or refresh, so +// pruneStaleShims can remove stale cache entries before the create/update loops in onNewRegistry +// run. It mirrors the filtering addRemoteCapabilities/serveCapabilities apply (unmarshal errors, +// LocalOnly, missing method config) but only collects keys - it never creates or mutates anything. +// Pass nil for remoteCapabilityDONs/myCapabilityDONs when the node doesn't currently belong to a +// workflow/capability DON, matching the gating in onNewRegistry. +func computeWantedShimKeys(remoteCapabilityDONs, myCapabilityDONs []registry.DON) wantedShimKeys { + wanted := wantedShimKeys{ + combinedClients: map[shimID]struct{}{}, + triggerSubscribers: map[shimID]struct{}{}, + executableClients: map[shimID]struct{}{}, + triggerPublishers: map[shimID]struct{}{}, + executableServers: map[shimID]struct{}{}, + } + for _, remoteDON := range remoteCapabilityDONs { + for cid, c := range remoteDON.CapabilityConfigurations { + methodConfig, ok := unmarshalWantedMethodConfig(c) + if !ok { + continue + } + wanted.combinedClients[shimID{capID: cid, donID: remoteDON.ID}] = struct{}{} + for method, cfg := range methodConfig { + key := shimID{capID: cid, donID: remoteDON.ID, method: method} + switch { + case cfg.RemoteTriggerConfig != nil: + wanted.triggerSubscribers[key] = struct{}{} + case cfg.RemoteExecutableConfig != nil: + wanted.executableClients[key] = struct{}{} + } + } + } + } + for _, myDON := range myCapabilityDONs { + for cid, c := range myDON.CapabilityConfigurations { + methodConfig, ok := unmarshalWantedMethodConfig(c) + if !ok { + continue + } + for method, cfg := range methodConfig { + key := shimID{capID: cid, donID: myDON.ID, method: method} + switch { + case cfg.RemoteTriggerConfig != nil: + wanted.triggerPublishers[key] = struct{}{} + case cfg.RemoteExecutableConfig != nil: + wanted.executableServers[key] = struct{}{} + } + } + } + } + return wanted +} + +func unmarshalWantedMethodConfig(c registry.CapabilityConfiguration) (map[string]capabilities.CapabilityMethodConfig, bool) { + capabilityConfig, err := c.Unmarshal() + if err != nil || capabilityConfig.LocalOnly || capabilityConfig.CapabilityMethodConfig == nil { + return nil, false + } + return capabilityConfig.CapabilityMethodConfig, true +} + +// pruneStaleShims tears down cached shims that are no longer in the wanted set: capabilities or +// methods removed from a DON's config, DONs that disappeared from the registry, or capabilities +// that moved to a different DON ID. It must run before the create/update loops in onNewRegistry - +// see computeWantedShimKeys. +func (w *launcher) pruneStaleShims(ctx context.Context, wanted wantedShimKeys) { + for key := range w.cachedShims.combinedClients { + if _, ok := wanted.combinedClients[key]; ok { + continue + } + if err := w.registry.Remove(ctx, key.capID); err != nil { + w.lggr.Errorw("failed to remove stale combined client from registry", "capabilityID", key.capID, "donID", key.donID, "error", err) + } + delete(w.cachedShims.combinedClients, key) + } + + for key, sub := range w.cachedShims.triggerSubscribers { + if _, ok := wanted.triggerSubscribers[key]; ok { + continue + } + w.closeStaleShim(key, sub) + delete(w.cachedShims.triggerSubscribers, key) + } + for key, client := range w.cachedShims.executableClients { + if _, ok := wanted.executableClients[key]; ok { + continue + } + w.closeStaleShim(key, client) + delete(w.cachedShims.executableClients, key) + } + for key, pub := range w.cachedShims.triggerPublishers { + if _, ok := wanted.triggerPublishers[key]; ok { + continue + } + w.closeStaleShim(key, pub) + delete(w.cachedShims.triggerPublishers, key) + } + for key, server := range w.cachedShims.executableServers { + if _, ok := wanted.executableServers[key]; ok { + continue + } + w.closeStaleShim(key, server) + delete(w.cachedShims.executableServers, key) + } +} + +// closeStaleShim deregisters a pruned shim from the dispatcher and closes it, removing it from +// w.subServices so the launcher's own Close() doesn't close it a second time. +func (w *launcher) closeStaleShim(key shimID, shim any) { + w.dispatcher.RemoveReceiverForMethod(key.capID, key.donID, key.method) + if c, ok := shim.(interface{ Close() error }); ok { + if err := c.Close(); err != nil { + w.lggr.Errorw("failed to close stale shim", "capabilityID", key.capID, "donID", key.donID, "method", key.method, "error", err) + } + } + + w.muSubServices.Lock() + defer w.muSubServices.Unlock() + for i, s := range w.subServices { + if any(s) == shim { + w.subServices = slices.Delete(w.subServices, i, i+1) + break + } + } +} diff --git a/core/capabilities/launcher_test.go b/core/capabilities/launcher_test.go index 426c2397b05..d7f91800281 100644 --- a/core/capabilities/launcher_test.go +++ b/core/capabilities/launcher_test.go @@ -638,6 +638,250 @@ func TestLauncher_V2CapabilitiesAddViaCombinedClient(t *testing.T) { require.NoError(t, err) assert.Equal(t, fullTriggerCapID, capInfo.ID) assert.Len(t, capInfo.DON.Members, 5) + + // The CombinedClient's own Info() (not just the per-method subscriber's) must also reflect + // the updated DON metadata, even though the DON ID - and therefore the cache key - is unchanged. + ccInfo, err := trigCC.Info(t.Context()) + require.NoError(t, err) + assert.Len(t, ccInfo.DON.Members, 5) +} + +func TestLauncher_OnNewRegistry_PrunesRemovedRemoteCapability(t *testing.T) { + t.Parallel() + lggr := logger.Test(t) + reg := regpkg.NewRegistry(lggr) + dispatcher := remoteMocks.NewDispatcher(t) + + workflowDonNodes, capabilityDonNodes := newNodes(4), newNodes(4) + fullTriggerCapID := "streams-trigger@1.0.0" + triggerCapIDHash := RandomUTF8BytesWord() + wfDonID := uint32(1) + capDonID := uint32(2) + + triggerCfg, err := proto.Marshal(&capabilitiespb.CapabilityConfig{ + MethodConfigs: map[string]*capabilitiespb.CapabilityMethodConfig{ + "StreamsTrigger": { + RemoteConfig: &capabilitiespb.CapabilityMethodConfig_RemoteTriggerConfig{ + RemoteTriggerConfig: &capabilitiespb.RemoteTriggerConfig{ + RegistrationRefresh: durationpb.New(1 * time.Second), + MinResponsesToAggregate: 3, + }, + }, + }, + }, + }) + require.NoError(t, err) + + localRegistry := buildLocalRegistry() + addDON(localRegistry, wfDonID, 0, 1, true, true, workflowDonNodes, []string{"zone-a"}, 1, nil) + addDON(localRegistry, capDonID, 0, 1, true, false, capabilityDonNodes, []string{"zone-a"}, 1, [][32]byte{triggerCapIDHash}) + addCapabilityToDON(localRegistry, capDonID, fullTriggerCapID, capabilities.CapabilityTypeTrigger, triggerCfg) + + sharedPeer := mocks.NewSharedPeer(t) + sharedPeer.On("ID").Return(workflowDonNodes[0]) + sharedPeer.On("IsBootstrap").Return(false) + sharedPeer.On("UpdateConnectionsByDONs", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + launcher, err := NewLauncher( + lggr, + sharedPeer, + nil, + dispatcher, + reg, + &mockDonNotifier{}, limits.Factory{}, + false, 0, + ) + require.NoError(t, err) + servicetest.Run(t, launcher) + + dispatcher.On("SetReceiverForMethod", fullTriggerCapID, capDonID, "StreamsTrigger", mock.AnythingOfType("*remote.triggerSubscriber")).Return(nil).Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + require.Len(t, launcher.cachedShims.triggerSubscribers, 1) + require.Len(t, launcher.cachedShims.combinedClients, 1) + + capObj, err := reg.Get(t.Context(), fullTriggerCapID) + require.NoError(t, err) + _, err = capObj.Info(t.Context()) + require.NoError(t, err) + + // The capability is removed entirely from the remote capability DON's config. + capDon := localRegistry.IDsToDONs[regpkg.DonID(capDonID)] + capDon.CapabilityConfigurations = map[string]regpkg.CapabilityConfiguration{} + localRegistry.IDsToDONs[regpkg.DonID(capDonID)] = capDon + delete(localRegistry.IDsToCapabilities, fullTriggerCapID) + + dispatcher.On("RemoveReceiverForMethod", fullTriggerCapID, capDonID, "StreamsTrigger").Return().Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + + assert.Empty(t, launcher.cachedShims.triggerSubscribers) + assert.Empty(t, launcher.cachedShims.combinedClients) + assert.Empty(t, launcher.subServices) + + // registry.Remove doesn't delete the map entry, it nils out the wrapped capability - so Get + // still succeeds, but the capability itself is now unavailable. + capObj, err = reg.Get(t.Context(), fullTriggerCapID) + require.NoError(t, err) + _, err = capObj.Info(t.Context()) + require.Error(t, err) +} + +func TestLauncher_OnNewRegistry_HandlesCapabilityDONReassignment(t *testing.T) { + t.Parallel() + lggr := logger.Test(t) + reg := regpkg.NewRegistry(lggr) + dispatcher := remoteMocks.NewDispatcher(t) + + workflowDonNodes := newNodes(4) + capabilityDonNodesA, capabilityDonNodesB := newNodes(4), newNodes(4) + fullTriggerCapID := "streams-trigger@1.0.0" + triggerCapIDHash := RandomUTF8BytesWord() + wfDonID := uint32(1) + capDonIDA := uint32(2) + capDonIDB := uint32(3) + + triggerCfg, err := proto.Marshal(&capabilitiespb.CapabilityConfig{ + MethodConfigs: map[string]*capabilitiespb.CapabilityMethodConfig{ + "StreamsTrigger": { + RemoteConfig: &capabilitiespb.CapabilityMethodConfig_RemoteTriggerConfig{ + RemoteTriggerConfig: &capabilitiespb.RemoteTriggerConfig{ + RegistrationRefresh: durationpb.New(1 * time.Second), + MinResponsesToAggregate: 3, + }, + }, + }, + }, + }) + require.NoError(t, err) + + localRegistry := buildLocalRegistry() + addDON(localRegistry, wfDonID, 0, 1, true, true, workflowDonNodes, []string{"zone-a"}, 1, nil) + addDON(localRegistry, capDonIDA, 0, 1, true, false, capabilityDonNodesA, []string{"zone-a"}, 1, [][32]byte{triggerCapIDHash}) + addCapabilityToDON(localRegistry, capDonIDA, fullTriggerCapID, capabilities.CapabilityTypeTrigger, triggerCfg) + + sharedPeer := mocks.NewSharedPeer(t) + sharedPeer.On("ID").Return(workflowDonNodes[0]) + sharedPeer.On("IsBootstrap").Return(false) + sharedPeer.On("UpdateConnectionsByDONs", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + launcher, err := NewLauncher( + lggr, + sharedPeer, + nil, + dispatcher, + reg, + &mockDonNotifier{}, limits.Factory{}, + false, 0, + ) + require.NoError(t, err) + servicetest.Run(t, launcher) + + dispatcher.On("SetReceiverForMethod", fullTriggerCapID, capDonIDA, "StreamsTrigger", mock.AnythingOfType("*remote.triggerSubscriber")).Return(nil).Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + require.Len(t, launcher.cachedShims.combinedClients, 1) + + // The capability moves from capDonIDA to a brand new capDonIDB, disappearing from A's config + // entirely in the same registry snapshot. + capDonA := localRegistry.IDsToDONs[regpkg.DonID(capDonIDA)] + capDonA.CapabilityConfigurations = map[string]regpkg.CapabilityConfiguration{} + localRegistry.IDsToDONs[regpkg.DonID(capDonIDA)] = capDonA + + addDON(localRegistry, capDonIDB, 0, 1, true, false, capabilityDonNodesB, []string{"zone-a"}, 1, [][32]byte{triggerCapIDHash}) + addCapabilityToDON(localRegistry, capDonIDB, fullTriggerCapID, capabilities.CapabilityTypeTrigger, triggerCfg) + + dispatcher.On("RemoveReceiverForMethod", fullTriggerCapID, capDonIDA, "StreamsTrigger").Return().Once() + dispatcher.On("SetReceiverForMethod", fullTriggerCapID, capDonIDB, "StreamsTrigger", mock.AnythingOfType("*remote.triggerSubscriber")).Return(nil).Once() + + // Must not fail with ErrCapabilityAlreadyExists: the stale combinedClient under capDonIDA has + // to be pruned (and removed from the registry) before the new one under capDonIDB is added. + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + + require.Len(t, launcher.cachedShims.combinedClients, 1) + for key := range launcher.cachedShims.combinedClients { + assert.Equal(t, capDonIDB, key.donID) + } + + capObj, err := reg.Get(t.Context(), fullTriggerCapID) + require.NoError(t, err) + info, err := capObj.Info(t.Context()) + require.NoError(t, err) + assert.Equal(t, capDonIDB, info.DON.ID) +} + +func TestLauncher_OnNewRegistry_PrunesRemovedServedCapability(t *testing.T) { + t.Parallel() + lggr := logger.Test(t) + reg := regpkg.NewRegistry(lggr) + fullTriggerCapID := "streams-trigger@1.0.0" + mt := newMockTrigger(capabilities.MustNewCapabilityInfo( + fullTriggerCapID, + capabilities.CapabilityTypeTrigger, + "streams trigger", + )) + require.NoError(t, reg.Add(t.Context(), mt)) + + dispatcher := remoteMocks.NewDispatcher(t) + + workflowDonNodes, capabilityDonNodes := newNodes(4), newNodes(4) + triggerCapID := RandomUTF8BytesWord() + wfDonID := uint32(1) + capDonID := uint32(2) + + triggerCfg, err := proto.Marshal(&capabilitiespb.CapabilityConfig{ + MethodConfigs: map[string]*capabilitiespb.CapabilityMethodConfig{ + "StreamsTrigger": { + RemoteConfig: &capabilitiespb.CapabilityMethodConfig_RemoteTriggerConfig{ + RemoteTriggerConfig: &capabilitiespb.RemoteTriggerConfig{ + RegistrationRefresh: durationpb.New(1 * time.Second), + MinResponsesToAggregate: 3, + }, + }, + }, + }, + }) + require.NoError(t, err) + + localRegistry := buildLocalRegistry() + addDON(localRegistry, wfDonID, 0, 1, true, true, workflowDonNodes, []string{"zone-a"}, 1, nil) + addDON(localRegistry, capDonID, 0, 1, true, false, capabilityDonNodes, []string{"zone-a"}, 1, [][32]byte{triggerCapID}) + addCapabilityToDON(localRegistry, capDonID, fullTriggerCapID, capabilities.CapabilityTypeTrigger, triggerCfg) + + sharedPeer := mocks.NewSharedPeer(t) + sharedPeer.On("ID").Return(capabilityDonNodes[0]) + sharedPeer.On("IsBootstrap").Return(false) + sharedPeer.On("UpdateConnectionsByDONs", mock.Anything, mock.Anything, mock.Anything).Return(nil) + + launcher, err := NewLauncher( + lggr, + sharedPeer, + nil, + dispatcher, + reg, + &mockDonNotifier{}, limits.Factory{}, + false, 0, + ) + require.NoError(t, err) + require.NoError(t, launcher.Start(t.Context())) + defer launcher.Close() + + dispatcher.On("SetReceiverForMethod", fullTriggerCapID, capDonID, "StreamsTrigger", mock.AnythingOfType("*remote.triggerPublisher")).Return(nil).Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + require.Len(t, launcher.cachedShims.triggerPublishers, 1) + + // The method is removed from the capability DON's config (e.g. the capability no longer hosts it). + capDon := localRegistry.IDsToDONs[regpkg.DonID(capDonID)] + capDon.CapabilityConfigurations = map[string]regpkg.CapabilityConfiguration{} + localRegistry.IDsToDONs[regpkg.DonID(capDonID)] = capDon + + dispatcher.On("RemoveReceiverForMethod", fullTriggerCapID, capDonID, "StreamsTrigger").Return().Once() + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + + assert.Empty(t, launcher.cachedShims.triggerPublishers) + assert.Empty(t, launcher.subServices) } func TestLauncher_V2CapabilitiesExposeRemotely(t *testing.T) { diff --git a/core/capabilities/remote/combined_client.go b/core/capabilities/remote/combined_client.go index 74447a6a5f1..4a0d9ec09c0 100644 --- a/core/capabilities/remote/combined_client.go +++ b/core/capabilities/remote/combined_client.go @@ -20,6 +20,7 @@ type CombinedClient interface { SetExecutableClient(method string, client capabilities.ExecutableCapability) GetTriggerSubscriber(method string) capabilities.TriggerCapability GetExecutableClient(method string) capabilities.ExecutableCapability + SetInfo(info capabilities.CapabilityInfo) } type combinedClient struct { @@ -32,6 +33,8 @@ type combinedClient struct { var _ CombinedClient = &combinedClient{} func (c *combinedClient) Info(ctx context.Context) (capabilities.CapabilityInfo, error) { + c.mu.RLock() + defer c.mu.RUnlock() return c.info, nil } @@ -118,3 +121,9 @@ func (c *combinedClient) GetExecutableClient(method string) capabilities.Executa defer c.mu.RUnlock() return c.executableClients[method] } + +func (c *combinedClient) SetInfo(info capabilities.CapabilityInfo) { + c.mu.Lock() + defer c.mu.Unlock() + c.info = info +} diff --git a/core/capabilities/remote/trigger_publisher.go b/core/capabilities/remote/trigger_publisher.go index 2ae1b02989d..dd18098e4f7 100644 --- a/core/capabilities/remote/trigger_publisher.go +++ b/core/capabilities/remote/trigger_publisher.go @@ -48,6 +48,7 @@ type triggerPublisher struct { bqMu sync.Mutex // protects batchingQueue stopCh services.StopChan wg sync.WaitGroup + closeOnce sync.Once ackExecutor *ParallelExecutor registerExecutor *ParallelExecutor lggr logger.Logger @@ -929,22 +930,27 @@ func (p *triggerPublisher) batchingLoop() { } func (p *triggerPublisher) Close() error { - close(p.stopCh) + var err error + p.closeOnce.Do(func() { + close(p.stopCh) - if p.ackExecutor != nil { - if err := p.ackExecutor.Close(); err != nil { - return fmt.Errorf("failed to close ack executor: %w", err) + if p.ackExecutor != nil { + if e := p.ackExecutor.Close(); e != nil { + err = fmt.Errorf("failed to close ack executor: %w", e) + return + } } - } - if p.registerExecutor != nil { - if err := p.registerExecutor.Close(); err != nil { - return fmt.Errorf("failed to close register executor: %w", err) + if p.registerExecutor != nil { + if e := p.registerExecutor.Close(); e != nil { + err = fmt.Errorf("failed to close register executor: %w", e) + return + } } - } - p.wg.Wait() - p.lggr.Info("TriggerPublisher closed") - return nil + p.wg.Wait() + p.lggr.Info("TriggerPublisher closed") + }) + return err } func (p *triggerPublisher) Ready() error { diff --git a/core/capabilities/remote/trigger_publisher_test.go b/core/capabilities/remote/trigger_publisher_test.go index 90d77b85bc1..5849aabfe71 100644 --- a/core/capabilities/remote/trigger_publisher_test.go +++ b/core/capabilities/remote/trigger_publisher_test.go @@ -522,7 +522,13 @@ func TestTriggerPublisher_ExplicitUnregister(t *testing.T) { } dispatcher := mocks.NewDispatcher(t) - allowRegistrationChecks(dispatcher) + checkReceived := make(chan *remotetypes.MessageBody, 10) + dispatcher.On("Send", mock.Anything, mock.MatchedBy(func(m *remotetypes.MessageBody) bool { + return m.Method == remotetypes.MethodTriggerRegistrationCheck + })).Run(func(args mock.Arguments) { + msg := args.Get(1).(*remotetypes.MessageBody) + checkReceived <- msg + }).Return(nil).Maybe() config := &commoncap.RemoteTriggerConfig{ RegistrationRefresh: 100 * time.Millisecond, @@ -549,6 +555,17 @@ func TestTriggerPublisher_ExplicitUnregister(t *testing.T) { waitForRegistration(t, underlying.registrationsCh) + // RegisterTrigger on the underlying capability runs asynchronously; the publisher only + // records the registration (and becomes eligible to process an unregister for it) after + // that call returns. Wait for a registration check, which is only sent for triggers + // already present in the publisher's registration map, so the race below can't drop the + // unregister message. + select { + case <-checkReceived: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for registration check message") + } + // Send unregister unregMsg := &remotetypes.MessageBody{ Sender: peers[1][:], diff --git a/core/capabilities/remote/trigger_subscriber.go b/core/capabilities/remote/trigger_subscriber.go index 3bcb0bbdc3e..192f4bda710 100644 --- a/core/capabilities/remote/trigger_subscriber.go +++ b/core/capabilities/remote/trigger_subscriber.go @@ -44,6 +44,7 @@ type triggerSubscriber struct { mu sync.RWMutex // protects registeredWorkflows, messageCache, and ackReplayCache stopCh services.StopChan wg sync.WaitGroup + closeOnce sync.Once lggr logger.Logger } @@ -495,9 +496,11 @@ func (s *triggerSubscriber) eventCleanupLoop() { } func (s *triggerSubscriber) Close() error { - close(s.stopCh) - s.wg.Wait() - s.lggr.Info("TriggerSubscriber closed") + s.closeOnce.Do(func() { + close(s.stopCh) + s.wg.Wait() + s.lggr.Info("TriggerSubscriber closed") + }) return nil } diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 8292efd61b5..acaa5c3c9e1 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -191,6 +191,9 @@ FeeBoost = false # Default [EVM.BalanceMonitor] # Enabled balance monitoring for all keys. Enabled = true # Default +# ERC20TokenAddress is the address of an ERC-20 type token that a chain supports as a fee token. +ERC20TokenAddress = "0x20C0000000000000000000000000000000000000" # Example + [EVM.GasEstimator] # Mode controls what type of gas estimator is used. diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 04d3b87f701..79da2f35895 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -33,10 +33,10 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.8 @@ -508,7 +508,7 @@ require ( github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index 913f3b39798..3874076ba55 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1683,20 +1683,20 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 125855c7f83..407c4735d39 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -1394,6 +1394,9 @@ func TestConfig_full(t *testing.T) { if got.EVM[c].GasEstimator.SenderAddress == nil { got.EVM[c].GasEstimator.SenderAddress = new(types.EIP55Address) } + if got.EVM[c].BalanceMonitor.ERC20TokenAddress == nil { + got.EVM[c].BalanceMonitor.ERC20TokenAddress = new(types.EIP55Address) + } } configtest.AssertFieldsNotNil(t, got) diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 1e9b2046986..834cb5d07c2 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -973,6 +973,9 @@ func newWorkflowRegistrySyncerV2( syncerV2.WithShardRoutingSteady(shardRoutingSteady), syncerV2.WithShardResolver(shardResolver), syncerV2.WithShardIndex(uint32(cfg.Sharding().ShardIndex())), + syncerV2.WithTriggerCoordinator( + syncerV2.NewTriggerCoordinator(opts.CapabilitiesRegistry, engineRegistry, clockwork.NewRealClock(), lggr), + ), } if shardingEnabled && dispatcher != nil { handlerOpts = append(handlerOpts, diff --git a/core/services/gateway/handler_factory.go b/core/services/gateway/handler_factory.go index a3755c16f6b..0034efa691a 100644 --- a/core/services/gateway/handler_factory.go +++ b/core/services/gateway/handler_factory.go @@ -2,7 +2,6 @@ package gateway import ( "encoding/json" - "errors" "fmt" "github.com/jonboulle/clockwork" @@ -65,29 +64,31 @@ func (hf *handlerFactory) NewHandler( shardedDONs []config.ShardedDONConfig, shardsConnMgrs [][]handlers.DON, ) (handlers.Handler, error) { - if len(shardedDONs) == 0 || len(shardsConnMgrs) == 0 { - return nil, errors.New("at least one DON and connection manager required") + dons, err := handlers.NewShardedDONs(shardedDONs, shardsConnMgrs) + if err != nil { + return nil, err } - if len(shardsConnMgrs[0]) == 0 { - return nil, errors.New("at least one shard connection manager required") - } - - // For backward compatibility, convert sharded config to legacy DONConfig - // using the first DON's first shard. Handlers that support sharding can - // use the full shardedDONs and shardsConnMgrs directly. - donConfig := shardedDONsToLegacy(shardedDONs[0]) - don := shardsConnMgrs[0][0] switch handlerType { case DummyHandlerType: - return handlers.NewDummyHandler(donConfig, don, hf.lggr) + return handlers.NewDummyHandler(dons, hf.lggr) case WebAPICapabilitiesType: - return capabilities.NewHandler(handlerConfig, donConfig, don, hf.httpClient, hf.lggr) + return capabilities.NewHandler(handlerConfig, dons, hf.httpClient, hf.lggr) case HTTPCapabilityType: - return v2.NewGatewayHandler(handlerConfig, shardedDONs, shardsConnMgrs, hf.httpClient, hf.lggr, hf.lf, hf.httpClientFactory, hf.orgResolver) + return v2.NewGatewayHandler(handlerConfig, dons, hf.httpClient, hf.lggr, hf.lf, hf.httpClientFactory, hf.orgResolver) case VaultHandlerType: + // For backward compatibility, convert sharded config to legacy DONConfig + // using the first DON's first shard. TODO(CRE-1640): migrate to full + // shardedDONs/shardsConnMgrs support. + donConfig := shardedDONsToLegacy(dons.DONs[0]) + don := dons.ConnMgrs[0][0] return vault.NewHandler(handlerConfig, donConfig, don, hf.capabilitiesRegistry, hf.workflowRegistrySyncer, hf.lggr, clockwork.NewRealClock(), hf.lf) case ConfidentialRelayHandlerType: + // For backward compatibility, convert sharded config to legacy DONConfig + // using the first DON's first shard. TODO(CRE-1640): migrate to full + // shardedDONs/shardsConnMgrs support. + donConfig := shardedDONsToLegacy(dons.DONs[0]) + don := dons.ConnMgrs[0][0] return confidentialrelay.NewHandler(handlerConfig, donConfig, don, hf.lggr, clockwork.NewRealClock(), hf.lf) default: return nil, fmt.Errorf("unsupported handler type %s", handlerType) diff --git a/core/services/gateway/handlers/capabilities/handler.go b/core/services/gateway/handlers/capabilities/handler.go index d7523f32abb..4e945313542 100644 --- a/core/services/gateway/handlers/capabilities/handler.go +++ b/core/services/gateway/handlers/capabilities/handler.go @@ -17,7 +17,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/ratelimit" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/common" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network" @@ -35,8 +34,7 @@ const ( type handler struct { services.StateMachine config HandlerConfig - don handlers.DON - donConfig *config.DONConfig + nodeAddrToShard map[string]*handlers.ShardEndpoint lggr logger.Logger httpClient network.HTTPClient nodeRateLimiter *ratelimit.RateLimiter @@ -50,13 +48,22 @@ type HandlerConfig struct { var _ handlers.Handler = (*handler)(nil) -func NewHandler(handlerConfig json.RawMessage, donConfig *config.DONConfig, don handlers.DON, httpClient network.HTTPClient, lggr logger.Logger) (*handler, error) { +func NewHandler(handlerConfig json.RawMessage, dons *handlers.ShardedDONs, httpClient network.HTTPClient, lggr logger.Logger) (*handler, error) { var cfg HandlerConfig err := json.Unmarshal(handlerConfig, &cfg) if err != nil { return nil, err } + _, nodeAddrToShard, err := dons.BuildShardEndpoints() + if err != nil { + return nil, err + } + defaultDonID := "" + if len(dons.DONs) > 0 { + defaultDonID = dons.DONs[0].DonName + } + nodeRateLimiter, err := ratelimit.NewRateLimiter(cfg.NodeRateLimiter) if err != nil { return nil, err @@ -69,9 +76,8 @@ func NewHandler(handlerConfig json.RawMessage, donConfig *config.DONConfig, don return &handler{ config: cfg, - don: don, - donConfig: donConfig, - lggr: logger.Named(lggr, "WebAPIHandler."+donConfig.DonID), + nodeAddrToShard: nodeAddrToShard, + lggr: logger.Named(lggr, "WebAPIHandler."+defaultDonID), httpClient: httpClient, nodeRateLimiter: nodeRateLimiter, metrics: metrics, @@ -181,7 +187,12 @@ func (h *handler) handleWebAPIOutgoingMessage(ctx context.Context, msg *api.Mess l.Errorw(ErrTransformingMessageToRequest, "err", err) return } - err = h.don.SendToNode(newCtx, nodeAddr, req) + shard, ok := h.nodeAddrToShard[nodeAddr] + if !ok { + l.Errorw("no connection manager found for node", "to", nodeAddr) + return + } + err = shard.ConnMgr.SendToNode(newCtx, nodeAddr, req) h.metrics.recordArtifactFetchResponseDelivery(newCtx, err == nil) if err != nil { l.Errorw("failed to send to node", "err", err, "to", nodeAddr) diff --git a/core/services/gateway/handlers/capabilities/handler_test.go b/core/services/gateway/handlers/capabilities/handler_test.go index 443fc9de0a4..854ae120796 100644 --- a/core/services/gateway/handlers/capabilities/handler_test.go +++ b/core/services/gateway/handlers/capabilities/handler_test.go @@ -16,6 +16,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" gwcommon "github.com/smartcontractkit/chainlink/v2/core/services/gateway/common" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" + "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" hc "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/common" handlermocks "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/mocks" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network" @@ -38,18 +39,23 @@ func setupHandler(t *testing.T) (*handler, *mocks.HTTPClient, *handlermocks.DON, cfgBytes, err := json.Marshal(handlerConfig) require.NoError(t, err) - donConfig := &config.DONConfig{ - Members: []config.NodeConfig{}, - F: 1, - } nodes := gwcommon.NewTestNodes(t, 2) + members := make([]config.NodeConfig, 0, len(nodes)) for id, n := range nodes { - donConfig.Members = append(donConfig.Members, config.NodeConfig{ + members = append(members, config.NodeConfig{ Name: fmt.Sprintf("node_%d", id), Address: n.Address, }) } - handler, err := NewHandler(json.RawMessage(cfgBytes), donConfig, don, httpClient, lggr) + shardedDONs := []config.ShardedDONConfig{ + { + F: 1, + Shards: []config.Shard{{Nodes: members}}, + }, + } + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{don}}) + require.NoError(t, err) + handler, err := NewHandler(json.RawMessage(cfgBytes), dons, httpClient, lggr) require.NoError(t, err) return handler, httpClient, don, nodes } diff --git a/core/services/gateway/handlers/capabilities/v2/http_handler.go b/core/services/gateway/handlers/capabilities/v2/http_handler.go index ceb6ba60c2b..5902a77a33b 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_handler.go +++ b/core/services/gateway/handlers/capabilities/v2/http_handler.go @@ -18,7 +18,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/capabilities/v2/metrics" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/network" @@ -46,8 +45,8 @@ const ( type gatewayHandler struct { services.StateMachine config ServiceConfig - shards []*shardEndpoint // all DON shards served by this handler, across the full DON×shard matrix - nodeAddrToShard map[string]*shardEndpoint // node address -> owning shard, for routing responses back to the correct shard conn manager + shards []*handlers.ShardEndpoint // all DON shards served by this handler, across the full DON×shard matrix + nodeAddrToShard map[string]*handlers.ShardEndpoint // node address -> owning shard, for routing responses back to the correct shard conn manager lggr logger.Logger httpClient network.HTTPClient wg sync.WaitGroup @@ -122,7 +121,7 @@ type RetryConfig struct { Multiplier float64 `json:"multiplier"` } -func NewGatewayHandler(handlerConfig json.RawMessage, shardedDONs []config.ShardedDONConfig, shardsConnMgrs [][]handlers.DON, httpClient network.HTTPClient, lggr logger.Logger, lf limits.Factory, httpClientFactory network.HTTPClientFactory, orgResolver orgresolver.OrgResolver) (*gatewayHandler, error) { +func NewGatewayHandler(handlerConfig json.RawMessage, dons *handlers.ShardedDONs, httpClient network.HTTPClient, lggr logger.Logger, lf limits.Factory, httpClientFactory network.HTTPClientFactory, orgResolver orgresolver.OrgResolver) (*gatewayHandler, error) { var cfg ServiceConfig err := json.Unmarshal(handlerConfig, &cfg) if err != nil { @@ -130,11 +129,11 @@ func NewGatewayHandler(handlerConfig json.RawMessage, shardedDONs []config.Shard } cfg = WithDefaults(cfg) - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, shardsConnMgrs) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() if err != nil { return nil, fmt.Errorf("failed to build shard endpoints: %w", err) } - members := allMembers(shards) + members := handlers.AllMembers(shards) globalNodeRateLimiter, err := lf.MakeRateLimiter(cresettings.Default.GatewayHTTPGlobalRate) if err != nil { @@ -614,11 +613,11 @@ func (h *gatewayHandler) sendResponseToNode(ctx context.Context, requestID strin if !ok { return fmt.Errorf("cannot route response to unknown node %s (no owning shard)", nodeAddr) } - err = shard.connMgr.SendToNode(ctx, nodeAddr, req) + err = shard.ConnMgr.SendToNode(ctx, nodeAddr, req) if err != nil { return err } - h.lggr.Debugw("sent response to node", "to", nodeAddr, "shard", shard.donID) + h.lggr.Debugw("sent response to node", "to", nodeAddr, "shard", shard.DonID) return nil } diff --git a/core/services/gateway/handlers/capabilities/v2/http_handler_test.go b/core/services/gateway/handlers/capabilities/v2/http_handler_test.go index f46fae5d4b6..703775280f2 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_handler_test.go +++ b/core/services/gateway/handlers/capabilities/v2/http_handler_test.go @@ -44,8 +44,8 @@ func TestNewGatewayHandler(t *testing.T) { mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) require.NotNil(t, handler.responseCache) @@ -60,8 +60,8 @@ func TestNewGatewayHandler(t *testing.T) { mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(invalidConfig, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(invalidConfig, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.Error(t, err) require.Nil(t, handler) }) @@ -78,8 +78,8 @@ func TestNewGatewayHandler(t *testing.T) { mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) require.Equal(t, defaultCleanUpPeriodMs, handler.config.CleanUpPeriodMs) // Default value @@ -90,7 +90,7 @@ func TestHandleNodeMessage(t *testing.T) { handler, _ := createTestHandler(t) t.Run("successful node message handling", func(t *testing.T) { - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) // Prepare outbound request @@ -131,7 +131,7 @@ func TestHandleNodeMessage(t *testing.T) { }) t.Run("successful node message handling with MultiHeaders", func(t *testing.T) { - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) // Prepare outbound request @@ -229,7 +229,7 @@ func TestHandleNodeMessage(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) // First call: should fetch from HTTP client and cache the response httpResp := &network.HTTPResponse{ StatusCode: 200, @@ -275,7 +275,7 @@ func TestHandleNodeMessage(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) httpResp := &network.HTTPResponse{ StatusCode: 500, @@ -446,8 +446,8 @@ func TestGatewayHandler_Start_CallsDeleteExpired(t *testing.T) { mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) mockCache := newMockResponseCache() @@ -475,10 +475,13 @@ func serviceCfg() ServiceConfig { // shardedArgs builds the multi-shard arguments for NewGatewayHandler from a // legacy single-shard DONConfig and its connection manager, producing a // one-DON one-shard matrix. -func shardedArgs(donConfig *config.DONConfig, mockDon *handlermocks.DON) ([]config.ShardedDONConfig, [][]handlers.DON) { - return []config.ShardedDONConfig{ +func shardedArgs(t *testing.T, donConfig *config.DONConfig, mockDon *handlermocks.DON) *handlers.ShardedDONs { + shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, - }, [][]handlers.DON{{mockDon}} + } + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + return dons } func createTestHandler(t *testing.T) (*gatewayHandler, *sdkmetric.ManualReader) { @@ -509,11 +512,11 @@ func createTestHandlerWithConfig(t *testing.T, cfg ServiceConfig) (*gatewayHandl mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - shardedDONs, connMgrs := shardedArgs(donConfig, mockDon) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + dons := shardedArgs(t, donConfig, mockDon) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) meter, reader := newBeholderReader(t) - handler.metrics, err = metrics.NewMetricsWithMeter(allMembers(handler.shards), meter) + handler.metrics, err = metrics.NewMetricsWithMeter(handlers.AllMembers(handler.shards), meter) require.NoError(t, err) require.NotNil(t, handler) @@ -700,7 +703,7 @@ func TestCreateHTTPRequestCallback(t *testing.T) { func TestMakeOutgoingRequest_SendResponseUsesIndependentContext(t *testing.T) { handler, _ := createTestHandler(t) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) outboundReq := gateway_common.OutboundHTTPRequest{ @@ -801,7 +804,7 @@ func TestHTTPActionLatencyMetrics(t *testing.T) { t.Run("async timer covers HTTP work plus response send; send has its own timer", func(t *testing.T) { t.Parallel() handler, reader := createTestHandler(t) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) const httpDelay = 120 * time.Millisecond @@ -843,7 +846,7 @@ func TestHTTPActionLatencyMetrics(t *testing.T) { t.Run("failed outbound call still records endpoint latency", func(t *testing.T) { t.Parallel() handler, reader := createTestHandler(t) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) const httpDelay = 100 * time.Millisecond @@ -869,7 +872,7 @@ func TestHTTPActionLatencyMetrics(t *testing.T) { t.Run("cache hit emits no outbound-call observation", func(t *testing.T) { t.Parallel() handler, reader := createTestHandler(t) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockCache := newMockResponseCache() mockCache.fetchResp = &gateway_common.OutboundHTTPResponse{StatusCode: 200} @@ -930,7 +933,7 @@ func TestMakeOutgoingRequestCachingBehavior(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) httpResp := &network.HTTPResponse{ StatusCode: 200, @@ -974,7 +977,7 @@ func TestMakeOutgoingRequestCachingBehavior(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) httpResp := &network.HTTPResponse{ StatusCode: 200, @@ -1018,7 +1021,7 @@ func TestMakeOutgoingRequestCachingBehavior(t *testing.T) { Result: &rawRequest, } - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) httpResp := &network.HTTPResponse{ StatusCode: 200, @@ -1060,7 +1063,7 @@ func setupRateLimitingTest(t *testing.T, cfg ServiceConfig) (*gatewayHandler, *j } mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) return handler, resp, mockHTTPClient, mockDon } @@ -1399,10 +1402,12 @@ func TestNewGatewayHandler_MultiShardCreatesRateLimitersForAllMembers(t *testing {handlermocks.NewDON(t), handlermocks.NewDON(t)}, {handlermocks.NewDON(t), handlermocks.NewDON(t)}, } + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) @@ -1410,10 +1415,10 @@ func TestNewGatewayHandler_MultiShardCreatesRateLimitersForAllMembers(t *testing require.Len(t, handler.shards, 4) // Verify donIDs: shard 0 = bare name, shard 1 = suffixed. - require.Equal(t, "donA", handler.shards[0].donID) - require.Equal(t, "donA_1", handler.shards[1].donID) - require.Equal(t, "donB", handler.shards[2].donID) - require.Equal(t, "donB_1", handler.shards[3].donID) + require.Equal(t, "donA", handler.shards[0].DonID) + require.Equal(t, "donA_1", handler.shards[1].DonID) + require.Equal(t, "donB", handler.shards[2].DonID) + require.Equal(t, "donB_1", handler.shards[3].DonID) // Verify per-node rate limiters created for all 8 members. require.Len(t, handler.perNodeRateLimiters, 8) @@ -1422,10 +1427,10 @@ func TestNewGatewayHandler_MultiShardCreatesRateLimitersForAllMembers(t *testing } // Verify nodeAddrToShard maps each node to the correct shard endpoint. - require.Equal(t, "donA", handler.nodeAddrToShard["a1"].donID) - require.Equal(t, "donA_1", handler.nodeAddrToShard["a3"].donID) - require.Equal(t, "donB", handler.nodeAddrToShard["b1"].donID) - require.Equal(t, "donB_1", handler.nodeAddrToShard["b3"].donID) + require.Equal(t, "donA", handler.nodeAddrToShard["a1"].DonID) + require.Equal(t, "donA_1", handler.nodeAddrToShard["a3"].DonID) + require.Equal(t, "donB", handler.nodeAddrToShard["b1"].DonID) + require.Equal(t, "donB_1", handler.nodeAddrToShard["b3"].DonID) } // TestGatewayHandler_SendResponseToNode_MultiShardRouting verifies that when a @@ -1449,10 +1454,12 @@ func TestGatewayHandler_SendResponseToNode_MultiShardRouting(t *testing.T) { shard0Don := handlermocks.NewDON(t) shard1Don := handlermocks.NewDON(t) connMgrs := [][]handlers.DON{{shard0Don, shard1Don}} + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) mockHTTPClient := httpmocks.NewHTTPClient(t) lggr := logger.Test(t) - handler, err := NewGatewayHandler(configBytes, shardedDONs, connMgrs, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) + handler, err := NewGatewayHandler(configBytes, dons, mockHTTPClient, lggr, limits.Factory{Logger: lggr}, defaultTestHTTPClientFactory, nil) require.NoError(t, err) require.NotNil(t, handler) diff --git a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go index ecb77ca6cac..f02c70db5ae 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go +++ b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler.go @@ -58,8 +58,8 @@ type savedCallback struct { type httpTriggerHandler struct { services.StateMachine config ServiceConfig - shards []*shardEndpoint - nodeAddrToShard map[string]*shardEndpoint + shards []*handlers.ShardEndpoint + nodeAddrToShard map[string]*handlers.ShardEndpoint lggr logger.Logger callbacksMu sync.Mutex callbacks map[string]savedCallback // requestID -> savedCallback @@ -77,7 +77,7 @@ type HTTPTriggerHandler interface { HandleNodeTriggerResponse(ctx context.Context, resp *jsonrpc.Response[json.RawMessage], nodeAddr string) error } -func NewHTTPTriggerHandler(lggr logger.Logger, cfg ServiceConfig, shards []*shardEndpoint, nodeAddrToShard map[string]*shardEndpoint, workflowMetadataHandler *WorkflowMetadataHandler, userRateLimiter limits.RateLimiter, metrics *metrics.Metrics, orgResolver orgresolver.OrgResolver) *httpTriggerHandler { +func NewHTTPTriggerHandler(lggr logger.Logger, cfg ServiceConfig, shards []*handlers.ShardEndpoint, nodeAddrToShard map[string]*handlers.ShardEndpoint, workflowMetadataHandler *WorkflowMetadataHandler, userRateLimiter limits.RateLimiter, metrics *metrics.Metrics, orgResolver orgresolver.OrgResolver) *httpTriggerHandler { return &httpTriggerHandler{ lggr: logger.Named(lggr, "RequestCallbacks"), callbacks: make(map[string]savedCallback), @@ -436,12 +436,12 @@ func (h *httpTriggerHandler) setupCallback(ctx context.Context, requestID string aggregators := make(map[string]*aggregation.IdenticalNodeResponseAggregator, len(assigned)) for _, shard := range assigned { // (N+F)//2 + 1 threshold where N = number of nodes, F = number of faulty nodes - threshold := (len(shard.members)+shard.f)/2 + 1 + threshold := (len(shard.Members)+shard.F)/2 + 1 agg, err := aggregation.NewIdenticalNodeResponseAggregator(threshold) if err != nil { return nil, errors.New("failed to create response aggregator: " + err.Error()) } - aggregators[shard.donID] = agg + aggregators[shard.DonID] = agg } doneCh := make(chan struct{}) @@ -489,18 +489,18 @@ func (h *httpTriggerHandler) HandleNodeTriggerResponse(ctx context.Context, resp if !ok { return fmt.Errorf("received trigger response from unknown node %s (no owning shard)", nodeAddr) } - agg, ok := saved.responseAggregators[shard.donID] + agg, ok := saved.responseAggregators[shard.DonID] if !ok { // The node belongs to a shard this workflow isn't assigned to (or the // callback was captured before the workflow was assigned there). - return fmt.Errorf("node %s (shard %s) is not assigned to workflow for request ID %s", nodeAddr, shard.donID, resp.ID) + return fmt.Errorf("node %s (shard %s) is not assigned to workflow for request ID %s", nodeAddr, shard.DonID, resp.ID) } aggResp, err := agg.CollectAndAggregate(resp, nodeAddr) if err != nil { return err } if aggResp == nil { - h.lggr.Debugw("Not enough responses to aggregate", "requestID", resp.ID, "nodeAddress", nodeAddr, "shard", shard.donID) + h.lggr.Debugw("Not enough responses to aggregate", "requestID", resp.ID, "nodeAddress", nodeAddr, "shard", shard.DonID) return nil } rawResp, err := json.Marshal(aggResp) @@ -683,7 +683,7 @@ func (h *httpTriggerHandler) sendWithRetries(ctx context.Context, legacyExecutio // sendToShard sends the request to all members of a single shard, retrying // failures until all succeed, the callback is responded to (doneCh), or ctx is // cancelled (overall max duration). -func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoint, legacyExecutionID, executionIDWithTriggerIndex string, req *jsonrpc.Request[json.RawMessage], doneCh <-chan struct{}) error { +func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *handlers.ShardEndpoint, legacyExecutionID, executionIDWithTriggerIndex string, req *jsonrpc.Request[json.RawMessage], doneCh <-chan struct{}) error { nodeTimeout := time.Duration(h.config.NodeSendTimeoutMs) * time.Millisecond successfulNodes := make(map[string]bool) @@ -696,7 +696,7 @@ func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoi for { var pending []string - for _, member := range shard.members { + for _, member := range shard.Members { if !successfulNodes[member.Address] { pending = append(pending, member.Address) } @@ -715,7 +715,7 @@ func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoi h.metrics.IncrementTriggerCapabilityRequestCount(ctx, nodeAddress, gateway_common.MethodWorkflowExecute, h.lggr) sendStart := time.Now() - err := shard.connMgr.SendToNode(nodeCtx, nodeAddress, req) + err := shard.ConnMgr.SendToNode(nodeCtx, nodeAddress, req) h.metrics.RecordGatewayToNodeLatency(ctx, time.Since(sendStart).Milliseconds(), nodeAddress, gateway_common.MethodWorkflowExecute, h.lggr) if err != nil { h.metrics.IncrementTriggerCapabilityRequestFailures(ctx, nodeAddress, gateway_common.MethodWorkflowExecute, h.lggr) @@ -733,7 +733,7 @@ func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoi combinedErr = errors.Join(combinedErr, fmt.Errorf("node %s: %w", res.nodeAddress, res.err)) h.lggr.Debugw("Failed to send trigger request to node, will retry", "node", res.nodeAddress, - "shard", shard.donID, + "shard", shard.DonID, "legacyExecutionID", legacyExecutionID, "executionIDWithTriggerIndex", executionIDWithTriggerIndex, "error", res.err) @@ -742,38 +742,38 @@ func (h *httpTriggerHandler) sendToShard(ctx context.Context, shard *shardEndpoi } } - if len(successfulNodes) == len(shard.members) { + if len(successfulNodes) == len(shard.Members) { h.lggr.Infow("Successfully sent trigger request to all nodes in shard", - "shard", shard.donID, + "shard", shard.DonID, "legacyExecutionID", legacyExecutionID, "executionIDWithTriggerIndex", executionIDWithTriggerIndex, - "nodeCount", len(shard.members)) + "nodeCount", len(shard.Members)) return nil } // Not all nodes succeeded, wait and retry h.lggr.Debugw("Retrying failed nodes for trigger request", - "shard", shard.donID, + "shard", shard.DonID, "legacyExecutionID", legacyExecutionID, "executionIDWithTriggerIndex", executionIDWithTriggerIndex, - "failedCount", len(shard.members)-len(successfulNodes), + "failedCount", len(shard.Members)-len(successfulNodes), "errors", combinedErr) select { case <-doneCh: h.lggr.Infow("Callback already responded to, stopping retries", - "shard", shard.donID, + "shard", shard.DonID, "legacyExecutionID", legacyExecutionID, "executionIDWithTriggerIndex", executionIDWithTriggerIndex, "requestID", req.ID, "successNodes", len(successfulNodes), - "totalNodes", len(shard.members)) + "totalNodes", len(shard.Members)) return nil case <-time.After(b.Duration()): continue case <-ctx.Done(): return fmt.Errorf("shard %s: request retry time exceeded, some nodes may not have received the request: legacyExecutionID=%s, executionIDWithTriggerIndex=%s, successNodes=%d, totalNodes=%d", - shard.donID, legacyExecutionID, executionIDWithTriggerIndex, len(successfulNodes), len(shard.members)) + shard.DonID, legacyExecutionID, executionIDWithTriggerIndex, len(successfulNodes), len(shard.Members)) } } } diff --git a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler_test.go b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler_test.go index e31d650d065..81127321a4b 100644 --- a/core/services/gateway/handlers/capabilities/v2/http_trigger_handler_test.go +++ b/core/services/gateway/handlers/capabilities/v2/http_trigger_handler_test.go @@ -545,7 +545,7 @@ func registerWorkflow(_ *testing.T, handler *httpTriggerHandler, workflowID stri func assignWorkflowToAllShards(mh *WorkflowMetadataHandler, workflowID string) { mh.mu.Lock() defer mh.mu.Unlock() - mh.workflowShards[workflowID] = append([]*shardEndpoint(nil), mh.shards...) + mh.workflowShards[workflowID] = append([]*handlers.ShardEndpoint(nil), mh.shards...) } // registerWorkflowOnShards is the sharding-aware variant of registerWorkflow. @@ -553,13 +553,13 @@ func assignWorkflowToAllShards(mh *WorkflowMetadataHandler, workflowID string) { // workflow to the given shard endpoints so that WorkflowShards(workflowID) // returns them and the trigger handler fans the request out only to those // shards. The metadata handler's map is written directly under its mutex. -func registerWorkflowOnShards(t *testing.T, handler *httpTriggerHandler, workflowID string, privateKey *ecdsa.PrivateKey, assignedShards ...*shardEndpoint) { +func registerWorkflowOnShards(t *testing.T, handler *httpTriggerHandler, workflowID string, privateKey *ecdsa.PrivateKey, assignedShards ...*handlers.ShardEndpoint) { t.Helper() registerWorkflow(t, handler, workflowID, privateKey) handler.workflowMetadataHandler.mu.Lock() defer handler.workflowMetadataHandler.mu.Unlock() - assigned := make([]*shardEndpoint, len(assignedShards)) + assigned := make([]*handlers.ShardEndpoint, len(assignedShards)) copy(assigned, assignedShards) handler.workflowMetadataHandler.workflowShards[workflowID] = assigned } @@ -933,15 +933,17 @@ func TestHttpTriggerHandler_HandleUserTriggerRequest_DeliversOnlyToRegisteredSha shard2Don := handlermocks.NewDON(t) connMgrs := [][]handlers.DON{{shard0Don, shard1Don, shard2Don}} - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, connMgrs) + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) require.Len(t, shards, 3) // Sanity-check donID derivation used by the metadata/trigger handlers. - require.Equal(t, "don", shards[0].donID) - require.Equal(t, "don_1", shards[1].donID) - require.Equal(t, "don_2", shards[2].donID) + require.Equal(t, "don", shards[0].DonID) + require.Equal(t, "don_1", shards[1].DonID) + require.Equal(t, "don_2", shards[2].DonID) - allMembersSlice := allMembers(shards) + allMembersSlice := handlers.AllMembers(shards) testMetrics, err := metrics.NewMetrics(allMembersSlice) require.NoError(t, err) metadataHandler := NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics) @@ -1005,9 +1007,9 @@ func TestHttpTriggerHandler_HandleUserTriggerRequest_JWTAuthorization(t *testing ctx := t.Context() // Setup metadata handler with test data - err := handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].donID].Start(ctx) + err := handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].donID].Close() + defer handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].DonID].Close() // Create test keys privateKey := createTestPrivateKey(t) @@ -1153,9 +1155,9 @@ func TestHttpTriggerHandler_HandleUserTriggerRequest_WorkflowLookup(t *testing.T handler, mockDon := createTestTriggerHandler(t) ctx := t.Context() - err := handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].donID].Start(ctx) + err := handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].donID].Close() + defer handler.workflowMetadataHandler.aggs[handler.workflowMetadataHandler.shards[0].DonID].Close() privateKey := createTestPrivateKey(t) signerAddr := crypto.PubkeyToAddress(privateKey.PublicKey) @@ -1838,7 +1840,9 @@ func createTestMetadataHandler(t *testing.T) *WorkflowMetadataHandler { shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) return NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics) } @@ -1851,7 +1855,9 @@ func newTestTriggerHandler(t *testing.T, lggr logger.Logger, cfg ServiceConfig, shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) // Repoint the metadata handler at THIS handler's shard set so that the // workflowShards populated by registerWorkflow route sends to mockDon. The @@ -1863,8 +1869,8 @@ func newTestTriggerHandler(t *testing.T, lggr logger.Logger, cfg ServiceConfig, metadataHandler.nodeAddrToShard = nodeAddrToShard metadataHandler.aggs = make(map[string]*aggregation.WorkflowMetadataAggregator, len(shards)) for _, shard := range shards { - threshold := shard.f + 1 - metadataHandler.aggs[shard.donID] = aggregation.NewWorkflowMetadataAggregator(metadataHandler.lggr, threshold, time.Duration(cfg.CleanUpPeriodMs)*time.Millisecond, testMetrics) + threshold := shard.F + 1 + metadataHandler.aggs[shard.DonID] = aggregation.NewWorkflowMetadataAggregator(metadataHandler.lggr, threshold, time.Duration(cfg.CleanUpPeriodMs)*time.Millisecond, testMetrics) } return NewHTTPTriggerHandler(lggr, cfg, shards, nodeAddrToShard, metadataHandler, userRateLimiter, testMetrics, nil) } @@ -1892,14 +1898,16 @@ func createTestTriggerHandlerWithConfig(t *testing.T, cfg ServiceConfig) (*httpT testMetrics := createTestMetrics(t, donConfig) // Build ONE shared shard set so the metadata handler and the trigger handler - // reference the SAME shardEndpoint instances (and the same mockDon). This is + // reference the SAME ShardEndpoint instances (and the same mockDon). This is // required because registerWorkflow writes workflowShards to the metadata // handler's .shards, and sendWithRetries reads them back and sends via - // shard.connMgr — both must hit mockDon, which the test sets expectations on. + // shard.ConnMgr — both must hit mockDon, which the test sets expectations on. shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) metadataHandler := NewWorkflowMetadataHandler(lggr, WithDefaults(cfg), shards, nodeAddrToShard, testMetrics) @@ -2223,7 +2231,7 @@ func TestHttpTriggerHandler_HandleUserTriggerRequest_StopsRetriesOnQuorum(t *tes // createMultiShardTriggerHandler builds a trigger handler over multiple shards // of a single DON. Each shard gets its own mock DON connection manager. Returns // the handler, the mock DONs (one per shard), and the shard endpoints. -func createMultiShardTriggerHandler(t *testing.T, donName string, shardNodeSets [][]string, f int) (*httpTriggerHandler, []*handlermocks.DON, []*shardEndpoint) { +func createMultiShardTriggerHandler(t *testing.T, donName string, shardNodeSets [][]string, f int) (*httpTriggerHandler, []*handlermocks.DON, []*handlers.ShardEndpoint) { t.Helper() lggr := logger.Test(t) cfg := WithDefaults(ServiceConfig{ @@ -2247,10 +2255,12 @@ func createMultiShardTriggerHandler(t *testing.T, donName string, shardNodeSets shardedDONs := []config.ShardedDONConfig{ {DonName: donName, F: f, Shards: shardsCfg}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{connMgrs}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{connMgrs}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) - allMembersSlice := allMembers(shards) + allMembersSlice := handlers.AllMembers(shards) testMetrics, err := metrics.NewMetrics(allMembersSlice) require.NoError(t, err) metadataHandler := NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics) @@ -2460,10 +2470,12 @@ func TestHttpTriggerHandler_MultiShardSendFailureResilience(t *testing.T) { shard0Don := handlermocks.NewDON(t) shard1Don := handlermocks.NewDON(t) connMgrs := [][]handlers.DON{{shard0Don, shard1Don}} - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, connMgrs) + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) - allMembersSlice := allMembers(shards) + allMembersSlice := handlers.AllMembers(shards) testMetrics, err := metrics.NewMetrics(allMembersSlice) require.NoError(t, err) metadataHandler := NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics) diff --git a/core/services/gateway/handlers/capabilities/v2/outbound_bounds_test.go b/core/services/gateway/handlers/capabilities/v2/outbound_bounds_test.go index 5c76b42e902..5ed77d3c82f 100644 --- a/core/services/gateway/handlers/capabilities/v2/outbound_bounds_test.go +++ b/core/services/gateway/handlers/capabilities/v2/outbound_bounds_test.go @@ -83,7 +83,7 @@ func TestMakeOutgoingRequest_NodeLimitIsolatesPeers(t *testing.T) { t.Parallel() handler, _ := createTestHandler(t) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient.EXPECT().Send(mock.Anything, mock.Anything).Return(okHTTPResponse(), nil).Once() mockDon.EXPECT().SendToNode(mock.Anything, "node2", mock.Anything).Return(nil) @@ -106,7 +106,7 @@ func TestMakeOutgoingRequest_ReleasesSlotOnCompletion(t *testing.T) { t.Parallel() handler, _ := createTestHandler(t) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient.EXPECT().Send(mock.Anything, mock.Anything).Return(okHTTPResponse(), nil).Times(3) mockDon.EXPECT().SendToNode(mock.Anything, "node1", mock.Anything).Return(nil) @@ -133,7 +133,7 @@ func TestMakeOutgoingRequest_ReleasesSlotOnFailure(t *testing.T) { t.Parallel() handler, _ := createTestHandler(t) mockHTTPClient := handler.httpClient.(*httpmocks.HTTPClient) - mockDon := handler.shards[0].connMgr.(*handlermocks.DON) + mockDon := handler.shards[0].ConnMgr.(*handlermocks.DON) mockHTTPClient.EXPECT().Send(mock.Anything, mock.Anything).Return(nil, errors.New("upstream down")).Once() mockDon.EXPECT().SendToNode(mock.Anything, "node1", mock.Anything).Return(nil) diff --git a/core/services/gateway/handlers/capabilities/v2/shard_endpoints.go b/core/services/gateway/handlers/capabilities/v2/shard_endpoints.go deleted file mode 100644 index 5404b86d9a8..00000000000 --- a/core/services/gateway/handlers/capabilities/v2/shard_endpoints.go +++ /dev/null @@ -1,77 +0,0 @@ -package v2 - -import ( - "errors" - "fmt" - - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" -) - -type shardEndpoint struct { - // donID is the shard-specific DON identifier, derived via config.ShardDONID. - // Shard 0 uses the bare DON name; shard N>0 uses "donName_N". - donID string - donName string - shardIdx int - // connMgr is the connection manager for this shard. It only knows how to reach - // nodes that are members of this shard. - connMgr handlers.DON - members []config.NodeConfig - f int -} - -// buildShardEndpoints expands the full DON×shard matrix into a flat list of -// shardEndpoints, and builds a nodeAddr -> shardEndpoint lookup. The outer -// dimension of shardsConnMgrs indexes DONs (parallel to shardedDONs); the inner -// dimension indexes shards within that DON (parallel to shardedDONs[i].Shards). -// Memberships must be disjoint. -func buildShardEndpoints(shardedDONs []config.ShardedDONConfig, shardsConnMgrs [][]handlers.DON) ([]*shardEndpoint, map[string]*shardEndpoint, error) { - if len(shardedDONs) == 0 || len(shardsConnMgrs) == 0 { - return nil, nil, errors.New("at least one DON and connection manager required") - } - if len(shardedDONs) != len(shardsConnMgrs) { - return nil, nil, fmt.Errorf("shardedDONs length %d does not match shardsConnMgrs length %d", len(shardedDONs), len(shardsConnMgrs)) - } - - var endpoints []*shardEndpoint - addrToShard := make(map[string]*shardEndpoint) - - for i, sd := range shardedDONs { - connMgrs := shardsConnMgrs[i] - if len(sd.Shards) != len(connMgrs) { - return nil, nil, fmt.Errorf("DON %s: shards length %d does not match connection managers length %d", sd.DonName, len(sd.Shards), len(connMgrs)) - } - for shardIdx, shard := range sd.Shards { - if connMgrs[shardIdx] == nil { - return nil, nil, fmt.Errorf("DON %s shard %d: nil connection manager", sd.DonName, shardIdx) - } - ep := &shardEndpoint{ - donID: config.ShardDONID(sd.DonName, shardIdx), - donName: sd.DonName, - shardIdx: shardIdx, - connMgr: connMgrs[shardIdx], - members: shard.Nodes, - f: sd.F, - } - endpoints = append(endpoints, ep) - for _, member := range shard.Nodes { - if existing, ok := addrToShard[member.Address]; ok { - return nil, nil, fmt.Errorf("node address %s appears in both %s and %s; shard memberships must be disjoint", member.Address, existing.donID, ep.donID) - } - addrToShard[member.Address] = ep - } - } - } - - return endpoints, addrToShard, nil -} - -// allMembers returns the union of all shard members across the matrix. -func allMembers(endpoints []*shardEndpoint) []config.NodeConfig { - var out []config.NodeConfig - for _, ep := range endpoints { - out = append(out, ep.members...) - } - return out -} diff --git a/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler.go b/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler.go index 43b2e39b4a9..609834361e7 100644 --- a/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler.go +++ b/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler.go @@ -14,6 +14,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/common/aggregation" + "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/capabilities/v2/metrics" "github.com/smartcontractkit/chainlink/v2/core/utils" ) @@ -40,11 +41,11 @@ type WorkflowMetadataHandler struct { authorizedKeys map[string]map[gateway.AuthorizedKey]struct{} // map of workflow ID to authorized keys workflowRefToID map[workflowReference]string // map of workflow reference to workflow ID workflowIDToRef map[string]workflowReference // map of workflow ID to workflow reference - workflowShards map[string][]*shardEndpoint // map of workflow ID to the shards it is assigned to (quorum reached) + workflowShards map[string][]*handlers.ShardEndpoint // map of workflow ID to the shards it is assigned to (quorum reached) // aggs holds one WorkflowMetadataAggregator per shard, keyed by shard donID. aggs map[string]*aggregation.WorkflowMetadataAggregator - shards []*shardEndpoint - nodeAddrToShard map[string]*shardEndpoint + shards []*handlers.ShardEndpoint + nodeAddrToShard map[string]*handlers.ShardEndpoint config ServiceConfig stopCh services.StopChan metrics *metrics.Metrics @@ -55,18 +56,18 @@ type WorkflowMetadataHandler struct { // NewWorkflowMetadataHandler creates a new WorkflowMetadataHandler spanning the // full DON×shard matrix. Each shard gets its own aggregator with threshold F+1. -func NewWorkflowMetadataHandler(lggr logger.Logger, cfg ServiceConfig, shards []*shardEndpoint, nodeAddrToShard map[string]*shardEndpoint, metrics *metrics.Metrics) *WorkflowMetadataHandler { +func NewWorkflowMetadataHandler(lggr logger.Logger, cfg ServiceConfig, shards []*handlers.ShardEndpoint, nodeAddrToShard map[string]*handlers.ShardEndpoint, metrics *metrics.Metrics) *WorkflowMetadataHandler { aggs := make(map[string]*aggregation.WorkflowMetadataAggregator, len(shards)) for _, shard := range shards { - threshold := shard.f + 1 - aggs[shard.donID] = aggregation.NewWorkflowMetadataAggregator(lggr, threshold, time.Duration(cfg.CleanUpPeriodMs)*time.Millisecond, metrics) + threshold := shard.F + 1 + aggs[shard.DonID] = aggregation.NewWorkflowMetadataAggregator(lggr, threshold, time.Duration(cfg.CleanUpPeriodMs)*time.Millisecond, metrics) } return &WorkflowMetadataHandler{ lggr: logger.Named(lggr, "HTTPTriggerWorkflowMetadataHandler"), authorizedKeys: make(map[string]map[gateway.AuthorizedKey]struct{}), workflowRefToID: make(map[workflowReference]string), workflowIDToRef: make(map[string]workflowReference), - workflowShards: make(map[string][]*shardEndpoint), + workflowShards: make(map[string][]*handlers.ShardEndpoint), aggs: aggs, shards: shards, nodeAddrToShard: nodeAddrToShard, @@ -115,10 +116,10 @@ func (h *WorkflowMetadataHandler) syncMetadata(ctx context.Context) { authorizedKeys := make(map[string]map[gateway.AuthorizedKey]struct{}) workflowRefToID := make(map[workflowReference]string) workflowIDToRef := make(map[string]workflowReference) - workflowShards := make(map[string][]*shardEndpoint) + workflowShards := make(map[string][]*handlers.ShardEndpoint) for _, shard := range h.shards { - agg := h.aggs[shard.donID] + agg := h.aggs[shard.DonID] metadata := agg.Aggregate() for _, data := range metadata { workflowID := data.WorkflowSelector.WorkflowID @@ -197,12 +198,12 @@ func (h *WorkflowMetadataHandler) sendMetadataPullRequest() error { } var combinedErr error for _, shard := range h.shards { - for _, member := range shard.members { + for _, member := range shard.Members { h.metrics.IncrementTriggerCapabilityRequestCount(ctx, member.Address, gateway.MethodPullWorkflowMetadata, h.lggr) - err := shard.connMgr.SendToNode(ctx, member.Address, req) + err := shard.ConnMgr.SendToNode(ctx, member.Address, req) if err != nil { h.metrics.IncrementTriggerCapabilityRequestFailures(ctx, member.Address, gateway.MethodPullWorkflowMetadata, h.lggr) - combinedErr = errors.Join(combinedErr, fmt.Errorf("failed to send pull request to node %s (shard %s): %w", member.Address, shard.donID, err)) + combinedErr = errors.Join(combinedErr, fmt.Errorf("failed to send pull request to node %s (shard %s): %w", member.Address, shard.DonID, err)) } } } @@ -263,16 +264,16 @@ func (h *WorkflowMetadataHandler) aggForNode(nodeAddr string) (*aggregation.Work if !ok { return nil, fmt.Errorf("received metadata from unknown node %s (no owning shard)", nodeAddr) } - return h.aggs[shard.donID], nil + return h.aggs[shard.DonID], nil } // WorkflowShards returns the shards a workflow is currently assigned to (those // whose quorum reported the workflow's metadata). The returned slice is a copy. -func (h *WorkflowMetadataHandler) WorkflowShards(workflowID string) []*shardEndpoint { +func (h *WorkflowMetadataHandler) WorkflowShards(workflowID string) []*handlers.ShardEndpoint { h.mu.RLock() defer h.mu.RUnlock() shards := h.workflowShards[workflowID] - out := make([]*shardEndpoint, len(shards)) + out := make([]*handlers.ShardEndpoint, len(shards)) copy(out, shards) return out } @@ -283,8 +284,8 @@ func (h *WorkflowMetadataHandler) Start(ctx context.Context) error { h.lggr.Info("Starting HTTP Trigger Metadata Handler") h.startTime = time.Now() for _, shard := range h.shards { - if err := h.aggs[shard.donID].Start(ctx); err != nil { - return fmt.Errorf("failed to start aggregator for shard %s: %w", shard.donID, err) + if err := h.aggs[shard.DonID].Start(ctx); err != nil { + return fmt.Errorf("failed to start aggregator for shard %s: %w", shard.DonID, err) } } h.runTicker(time.Duration(h.config.MetadataPullIntervalMs)*time.Millisecond, func(ctx context.Context) { @@ -379,8 +380,8 @@ func (h *WorkflowMetadataHandler) Close() error { return h.StopOnce("WorkflowMetadataHandler", func() error { h.lggr.Info("Stopping HTTP Trigger Metadata Handler") for _, shard := range h.shards { - if err := h.aggs[shard.donID].Close(); err != nil { - h.lggr.Errorw("Failed to close WorkflowMetadataAggregator", "shard", shard.donID, "error", err) + if err := h.aggs[shard.DonID].Close(); err != nil { + h.lggr.Errorw("Failed to close WorkflowMetadataAggregator", "shard", shard.DonID, "error", err) } } close(h.stopCh) diff --git a/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler_test.go b/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler_test.go index 42444fb96a1..ac861778a3f 100644 --- a/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler_test.go +++ b/core/services/gateway/handlers/capabilities/v2/workflow_metadata_handler_test.go @@ -59,11 +59,13 @@ func createTestWorkflowMetadataHandler(t *testing.T) (*WorkflowMetadataHandler, // singleShardEndpoints builds a one-DON one-shard endpoint matrix from a legacy // DONConfig for tests. -func singleShardEndpoints(t *testing.T, donConfig *config.DONConfig, mockDon *mocks.DON) ([]*shardEndpoint, map[string]*shardEndpoint) { +func singleShardEndpoints(t *testing.T, donConfig *config.DONConfig, mockDon *mocks.DON) ([]*handlers.ShardEndpoint, map[string]*handlers.ShardEndpoint) { shardedDONs := []config.ShardedDONConfig{ {DonName: donConfig.DonID, F: donConfig.F, Shards: []config.Shard{{Nodes: donConfig.Members}}}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) return shards, nodeAddrToShard } @@ -77,9 +79,9 @@ func TestSyncMetadata(t *testing.T) { // Start the aggregator to enable data collection ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() // Add some test data to aggregator key := gateway_common.AuthorizedKey{ @@ -97,9 +99,9 @@ func TestSyncMetadata(t *testing.T) { } // Collect enough observations to meet threshold (F+1 = 2) - err = handler.aggs[handler.shards[0].donID].Collect(&observation, "node1") + err = handler.aggs[handler.shards[0].DonID].Collect(&observation, "node1") require.NoError(t, err) - err = handler.aggs[handler.shards[0].donID].Collect(&observation, "node2") + err = handler.aggs[handler.shards[0].DonID].Collect(&observation, "node2") require.NoError(t, err) handler.syncMetadata(t.Context()) @@ -125,9 +127,9 @@ func TestSyncMetadataMultipleWorkflows(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() // Add observations for multiple workflows workflows := []string{"workflow1", "workflow2"} @@ -149,9 +151,9 @@ func TestSyncMetadataMultipleWorkflows(t *testing.T) { }, }, } - err = handler.aggs[handler.shards[0].donID].Collect(&observation, "node1") + err = handler.aggs[handler.shards[0].DonID].Collect(&observation, "node1") require.NoError(t, err) - err = handler.aggs[handler.shards[0].donID].Collect(&observation, "node2") + err = handler.aggs[handler.shards[0].DonID].Collect(&observation, "node2") require.NoError(t, err) } } @@ -248,9 +250,11 @@ func createMultiShardMetadataHandler(t *testing.T) (*WorkflowMetadataHandler, *m {Nodes: []config.NodeConfig{{Address: "node3"}, {Address: "node4"}}}, }}, } - shards, nodeAddrToShard, err := buildShardEndpoints(shardedDONs, [][]handlers.DON{{mockDon0, mockDon1}}) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{mockDon0, mockDon1}}) + require.NoError(t, err) + shards, nodeAddrToShard, err := dons.BuildShardEndpoints() require.NoError(t, err) - testMetrics, err := metrics.NewMetrics(allMembers(shards)) + testMetrics, err := metrics.NewMetrics(handlers.AllMembers(shards)) require.NoError(t, err) cfg := WithDefaults(ServiceConfig{}) return NewWorkflowMetadataHandler(lggr, cfg, shards, nodeAddrToShard, testMetrics), mockDon0, mockDon1 @@ -262,8 +266,8 @@ func TestMultiShardAssignment(t *testing.T) { handler, _, _ := createMultiShardMetadataHandler(t) ctx := t.Context() for _, shard := range handler.shards { - require.NoError(t, handler.aggs[shard.donID].Start(ctx)) - t.Cleanup(func() { _ = handler.aggs[shard.donID].Close() }) + require.NoError(t, handler.aggs[shard.DonID].Start(ctx)) + t.Cleanup(func() { _ = handler.aggs[shard.DonID].Close() }) } obs := gateway_common.WorkflowMetadata{ @@ -280,14 +284,14 @@ func TestMultiShardAssignment(t *testing.T) { require.Empty(t, handler.WorkflowShards(testWorkflowID1)) // Quorum (F+1=1) reached only in shard 1 (node3). - require.NoError(t, handler.aggs[handler.shards[1].donID].Collect(&obs, "node3")) + require.NoError(t, handler.aggs[handler.shards[1].DonID].Collect(&obs, "node3")) handler.syncMetadata(ctx) assigned := handler.WorkflowShards(testWorkflowID1) require.Len(t, assigned, 1) - require.Equal(t, handler.shards[1].donID, assigned[0].donID) + require.Equal(t, handler.shards[1].DonID, assigned[0].DonID) // Later quorum reached in shard 0 too (node1) => assigned to both shards. - require.NoError(t, handler.aggs[handler.shards[0].donID].Collect(&obs, "node1")) + require.NoError(t, handler.aggs[handler.shards[0].DonID].Collect(&obs, "node1")) handler.syncMetadata(ctx) assigned = handler.WorkflowShards(testWorkflowID1) require.Len(t, assigned, 2) @@ -316,9 +320,9 @@ func TestOnMetadataPush(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() metadata := gateway_common.WorkflowMetadata{ WorkflowSelector: gateway_common.WorkflowSelector{ @@ -374,9 +378,9 @@ func TestOnMetadataPullResponse(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() key1 := gateway_common.AuthorizedKey{ KeyType: gateway_common.KeyTypeECDSAEVM, @@ -903,9 +907,9 @@ func TestOnMetadataPushWithValidation(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() t.Run("valid metadata passes validation", func(t *testing.T) { metadata := gateway_common.WorkflowMetadata{ @@ -969,9 +973,9 @@ func TestOnMetadataPullResponseWithValidation(t *testing.T) { handler, _, _ := createTestWorkflowMetadataHandler(t) ctx := t.Context() - err := handler.aggs[handler.shards[0].donID].Start(ctx) + err := handler.aggs[handler.shards[0].DonID].Start(ctx) require.NoError(t, err) - defer handler.aggs[handler.shards[0].donID].Close() + defer handler.aggs[handler.shards[0].DonID].Close() t.Run("valid metadata array passes validation", func(t *testing.T) { metadata := []gateway_common.WorkflowMetadata{ diff --git a/core/services/gateway/handlers/handler.dummy.go b/core/services/gateway/handlers/handler.dummy.go index ecc991627c6..f525db2a887 100644 --- a/core/services/gateway/handlers/handler.dummy.go +++ b/core/services/gateway/handlers/handler.dummy.go @@ -10,13 +10,12 @@ import ( jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" - "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" ) // DummyHandler forwards each request/response without doing any checks. type dummyHandler struct { - donConfig *config.DONConfig - don DON + shards []*ShardEndpoint + defaultDonID string savedCallbacks map[string]*savedCallback mu sync.Mutex lggr logger.Logger @@ -29,12 +28,23 @@ type savedCallback struct { var _ Handler = (*dummyHandler)(nil) -func NewDummyHandler(donConfig *config.DONConfig, don DON, lggr logger.Logger) (Handler, error) { +// NewDummyHandler builds a handler that forwards each request to every node across +// every DON and shard it is given, and relays the first node response back to the +// caller. +func NewDummyHandler(dons *ShardedDONs, lggr logger.Logger) (Handler, error) { + shards, _, err := dons.BuildShardEndpoints() + if err != nil { + return nil, err + } + defaultDonID := "" + if len(dons.DONs) > 0 { + defaultDonID = dons.DONs[0].DonName + } return &dummyHandler{ - donConfig: donConfig, - don: don, + shards: shards, + defaultDonID: defaultDonID, savedCallbacks: make(map[string]*savedCallback), - lggr: logger.Named(lggr, "DummyHandler."+donConfig.DonID), + lggr: logger.Named(lggr, "DummyHandler."+defaultDonID), }, nil } @@ -54,7 +64,7 @@ func (d *dummyHandler) HandleJSONRPCUserMessage(ctx context.Context, jsonRequest msg.Body.Method = jsonRequest.Method } if msg.Body.DonID == "" { - msg.Body.DonID = d.donConfig.DonID + msg.Body.DonID = d.defaultDonID } return d.HandleLegacyUserMessage(ctx, &msg, callback) } @@ -62,7 +72,6 @@ func (d *dummyHandler) HandleJSONRPCUserMessage(ctx context.Context, jsonRequest func (d *dummyHandler) HandleLegacyUserMessage(ctx context.Context, msg *api.Message, callback Callback) error { d.mu.Lock() d.savedCallbacks[msg.Body.MessageID] = &savedCallback{msg.Body.MessageID, callback} - don := d.don d.mu.Unlock() params, err := json.Marshal(msg) if err != nil { @@ -75,8 +84,10 @@ func (d *dummyHandler) HandleLegacyUserMessage(ctx context.Context, msg *api.Mes Method: msg.Body.Method, Params: &rawParams, } - for _, member := range d.donConfig.Members { - err = errors.Join(err, don.SendToNode(ctx, member.Address, req)) + for _, shard := range d.shards { + for _, member := range shard.Members { + err = errors.Join(err, shard.ConnMgr.SendToNode(ctx, member.Address, req)) + } } return err } diff --git a/core/services/gateway/handlers/handler.dummy_test.go b/core/services/gateway/handlers/handler.dummy_test.go index e69bdfced28..6c1e470d9c8 100644 --- a/core/services/gateway/handlers/handler.dummy_test.go +++ b/core/services/gateway/handlers/handler.dummy_test.go @@ -37,15 +37,24 @@ func (m *testConnManager) SendToNode(ctx context.Context, nodeAddress string, re func TestDummyHandler_BasicFlow(t *testing.T) { t.Parallel() - config := config.DONConfig{ - Members: []config.NodeConfig{ - {Name: "node one", Address: "addr_1"}, - {Name: "node two", Address: "addr_2"}, + shardedDONs := []config.ShardedDONConfig{ + { + DonName: "test_don", + Shards: []config.Shard{ + { + Nodes: []config.NodeConfig{ + {Name: "node one", Address: "addr_1"}, + {Name: "node two", Address: "addr_2"}, + }, + }, + }, }, } connMgr := testConnManager{} - handler, err := handlers.NewDummyHandler(&config, &connMgr, logger.Test(t)) + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{&connMgr}}) + require.NoError(t, err) + handler, err := handlers.NewDummyHandler(dons, logger.Test(t)) require.NoError(t, err) connMgr.SetHandler("", handler) diff --git a/core/services/gateway/handlers/shard_endpoints.go b/core/services/gateway/handlers/shard_endpoints.go new file mode 100644 index 00000000000..09cd242fa04 --- /dev/null +++ b/core/services/gateway/handlers/shard_endpoints.go @@ -0,0 +1,99 @@ +package handlers + +import ( + "errors" + "fmt" + + "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" +) + +// ShardedDONs pairs a DON×shard config matrix with the connection managers that +// serve it. The outer dimension of ConnMgrs indexes DONs (parallel to DONs); the +// inner dimension indexes shards within that DON (parallel to DONs[i].Shards). +// Use NewShardedDONs to build one: it checks the two stay in lockstep once, so +// callers can pass a single value around instead of two slices that must always +// agree in length. +type ShardedDONs struct { + DONs []config.ShardedDONConfig + ConnMgrs [][]DON +} + +// NewShardedDONs validates that shardedDONs and shardsConnMgrs describe the same +// DON×shard matrix (equal outer length, equal per-DON shard counts, no nil +// connection managers) and bundles them together. +func NewShardedDONs(shardedDONs []config.ShardedDONConfig, shardsConnMgrs [][]DON) (*ShardedDONs, error) { + if len(shardedDONs) == 0 || len(shardsConnMgrs) == 0 { + return nil, errors.New("at least one DON and connection manager required") + } + if len(shardedDONs) != len(shardsConnMgrs) { + return nil, fmt.Errorf("shardedDONs length %d does not match shardsConnMgrs length %d", len(shardedDONs), len(shardsConnMgrs)) + } + for i, sd := range shardedDONs { + connMgrs := shardsConnMgrs[i] + if len(sd.Shards) != len(connMgrs) { + return nil, fmt.Errorf("DON %s: shards length %d does not match connection managers length %d", sd.DonName, len(sd.Shards), len(connMgrs)) + } + for shardIdx, connMgr := range connMgrs { + if connMgr == nil { + return nil, fmt.Errorf("DON %s shard %d: nil connection manager", sd.DonName, shardIdx) + } + } + } + return &ShardedDONs{DONs: shardedDONs, ConnMgrs: shardsConnMgrs}, nil +} + +// ShardEndpoint is one shard of one DON: its identifier, the connection manager +// that reaches its members, and the config (membership, Byzantine fault +// tolerance threshold F) that applies to it. +type ShardEndpoint struct { + // DonID is the shard-specific DON identifier, derived via config.ShardDONID. + // Shard 0 uses the bare DON name; shard N>0 uses "donName_N". + DonID string + DonName string + ShardIdx int + // ConnMgr is the connection manager for this shard. It only knows how to reach + // nodes that are members of this shard. + ConnMgr DON + Members []config.NodeConfig + F int +} + +// BuildShardEndpoints expands the DON×shard matrix into a flat list of +// ShardEndpoints, and builds a nodeAddr -> ShardEndpoint lookup. Node addresses +// must be unique (disjoint) across the whole matrix. +func (s *ShardedDONs) BuildShardEndpoints() ([]*ShardEndpoint, map[string]*ShardEndpoint, error) { + var endpoints []*ShardEndpoint + addrToShard := make(map[string]*ShardEndpoint) + + for i, sd := range s.DONs { + connMgrs := s.ConnMgrs[i] + for shardIdx, shard := range sd.Shards { + ep := &ShardEndpoint{ + DonID: config.ShardDONID(sd.DonName, shardIdx), + DonName: sd.DonName, + ShardIdx: shardIdx, + ConnMgr: connMgrs[shardIdx], + Members: shard.Nodes, + F: sd.F, + } + endpoints = append(endpoints, ep) + for _, member := range shard.Nodes { + if existing, ok := addrToShard[member.Address]; ok { + return nil, nil, fmt.Errorf("node address %s appears in both %s and %s; shard memberships must be disjoint", member.Address, existing.DonID, ep.DonID) + } + addrToShard[member.Address] = ep + } + } + } + + return endpoints, addrToShard, nil +} + +// AllMembers returns the union of all shard members across the matrix. +func AllMembers(endpoints []*ShardEndpoint) []config.NodeConfig { + var out []config.NodeConfig + for _, ep := range endpoints { + out = append(out, ep.Members...) + } + return out +} diff --git a/core/services/gateway/handlers/capabilities/v2/shard_endpoints_test.go b/core/services/gateway/handlers/shard_endpoints_test.go similarity index 58% rename from core/services/gateway/handlers/capabilities/v2/shard_endpoints_test.go rename to core/services/gateway/handlers/shard_endpoints_test.go index 01dd876468c..d782af55578 100644 --- a/core/services/gateway/handlers/capabilities/v2/shard_endpoints_test.go +++ b/core/services/gateway/handlers/shard_endpoints_test.go @@ -1,4 +1,4 @@ -package v2 +package handlers_test import ( "testing" @@ -10,15 +10,58 @@ import ( handlermocks "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/mocks" ) -// handlersDON is a local alias to keep the test matrix literals readable. -type handlersDON = handlers.DON +func node(addr string) config.NodeConfig { + return config.NodeConfig{Name: addr, Address: addr} +} -func TestBuildShardEndpoints(t *testing.T) { +func TestNewShardedDONs(t *testing.T) { t.Parallel() - node := func(addr string) config.NodeConfig { - return config.NodeConfig{Name: addr, Address: addr} - } + t.Run("rejects empty matrix", func(t *testing.T) { + t.Parallel() + + _, err := handlers.NewShardedDONs(nil, nil) + require.Error(t, err) + }) + + t.Run("rejects ragged outer dimension", func(t *testing.T) { + t.Parallel() + + shardedDONs := []config.ShardedDONConfig{ + {DonName: "donA", Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1")}}}}, + } + _, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{}) + require.Error(t, err) + }) + + t.Run("rejects ragged inner dimension", func(t *testing.T) { + t.Parallel() + + shardedDONs := []config.ShardedDONConfig{ + {DonName: "donA", Shards: []config.Shard{ + {Nodes: []config.NodeConfig{node("n1")}}, + {Nodes: []config.NodeConfig{node("n2")}}, + }}, + } + connMgrs := [][]handlers.DON{{handlermocks.NewDON(t)}} // only 1 conn mgr for 2 shards + _, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.Error(t, err) + }) + + t.Run("rejects nil connection manager", func(t *testing.T) { + t.Parallel() + + shardedDONs := []config.ShardedDONConfig{ + {DonName: "donA", Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1")}}}}, + } + connMgrs := [][]handlers.DON{{nil}} + _, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.Error(t, err) + }) +} + +func TestShardedDONs_BuildShardEndpoints(t *testing.T) { + t.Parallel() t.Run("single DON single shard (backward compatible)", func(t *testing.T) { t.Parallel() @@ -27,14 +70,15 @@ func TestBuildShardEndpoints(t *testing.T) { shardedDONs := []config.ShardedDONConfig{ {DonName: "donA", F: 1, Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1"), node("n2")}}}}, } - connMgrs := [][]handlersDON{{don}} + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{don}}) + require.NoError(t, err) - eps, addrMap, err := buildShardEndpoints(shardedDONs, connMgrs) + eps, addrMap, err := dons.BuildShardEndpoints() require.NoError(t, err) require.Len(t, eps, 1) - require.Equal(t, "donA", eps[0].donID) // shard 0 => bare name - require.Equal(t, 0, eps[0].shardIdx) - require.Equal(t, 1, eps[0].f) + require.Equal(t, "donA", eps[0].DonID) // shard 0 => bare name + require.Equal(t, 0, eps[0].ShardIdx) + require.Equal(t, 1, eps[0].F) require.Len(t, addrMap, 2) require.Same(t, eps[0], addrMap["n1"]) }) @@ -42,7 +86,7 @@ func TestBuildShardEndpoints(t *testing.T) { t.Run("full matrix: two DONs each with two shards", func(t *testing.T) { t.Parallel() - dons := [][]handlersDON{ + connMgrs := [][]handlers.DON{ {handlermocks.NewDON(t), handlermocks.NewDON(t)}, {handlermocks.NewDON(t), handlermocks.NewDON(t)}, } @@ -56,86 +100,56 @@ func TestBuildShardEndpoints(t *testing.T) { {Nodes: []config.NodeConfig{node("b1")}}, }}, } + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) - eps, addrMap, err := buildShardEndpoints(shardedDONs, dons) + eps, addrMap, err := dons.BuildShardEndpoints() require.NoError(t, err) require.Len(t, eps, 4) // donIDs: shard 0 bare, shard 1 suffixed - require.Equal(t, "donA", eps[0].donID) - require.Equal(t, "donA_1", eps[1].donID) - require.Equal(t, "donB", eps[2].donID) - require.Equal(t, "donB_1", eps[3].donID) + require.Equal(t, "donA", eps[0].DonID) + require.Equal(t, "donA_1", eps[1].DonID) + require.Equal(t, "donB", eps[2].DonID) + require.Equal(t, "donB_1", eps[3].DonID) require.Len(t, addrMap, 6) require.Same(t, eps[1], addrMap["a2"]) require.Same(t, eps[3], addrMap["b1"]) - require.Equal(t, 2, addrMap["b1"].f) + require.Equal(t, 2, addrMap["b1"].F) }) - t.Run("rejects empty matrix", func(t *testing.T) { - t.Parallel() - - _, _, err := buildShardEndpoints(nil, nil) - require.Error(t, err) - }) - - t.Run("rejects ragged outer dimension", func(t *testing.T) { - t.Parallel() - - shardedDONs := []config.ShardedDONConfig{ - {DonName: "donA", Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1")}}}}, - } - _, _, err := buildShardEndpoints(shardedDONs, [][]handlersDON{}) - require.Error(t, err) - }) - - t.Run("rejects ragged inner dimension", func(t *testing.T) { + t.Run("rejects duplicate node address across shards", func(t *testing.T) { t.Parallel() shardedDONs := []config.ShardedDONConfig{ {DonName: "donA", Shards: []config.Shard{ {Nodes: []config.NodeConfig{node("n1")}}, - {Nodes: []config.NodeConfig{node("n2")}}, + {Nodes: []config.NodeConfig{node("n1")}}, // duplicate }}, } - connMgrs := [][]handlersDON{{handlermocks.NewDON(t)}} // only 1 conn mgr for 2 shards - _, _, err := buildShardEndpoints(shardedDONs, connMgrs) - require.Error(t, err) - }) - - t.Run("rejects nil connection manager", func(t *testing.T) { - t.Parallel() + connMgrs := [][]handlers.DON{{handlermocks.NewDON(t), handlermocks.NewDON(t)}} + dons, err := handlers.NewShardedDONs(shardedDONs, connMgrs) + require.NoError(t, err) - shardedDONs := []config.ShardedDONConfig{ - {DonName: "donA", Shards: []config.Shard{{Nodes: []config.NodeConfig{node("n1")}}}}, - } - connMgrs := [][]handlersDON{{nil}} - _, _, err := buildShardEndpoints(shardedDONs, connMgrs) + _, _, err = dons.BuildShardEndpoints() require.Error(t, err) + require.Contains(t, err.Error(), "disjoint") }) - t.Run("rejects duplicate node address across shards", func(t *testing.T) { + t.Run("AllMembers unions all shards", func(t *testing.T) { t.Parallel() + don0, don1 := handlermocks.NewDON(t), handlermocks.NewDON(t) shardedDONs := []config.ShardedDONConfig{ {DonName: "donA", Shards: []config.Shard{ - {Nodes: []config.NodeConfig{node("n1")}}, - {Nodes: []config.NodeConfig{node("n1")}}, // duplicate + {Nodes: []config.NodeConfig{node("n1"), node("n2")}}, + {Nodes: []config.NodeConfig{node("n3")}}, }}, } - connMgrs := [][]handlersDON{{handlermocks.NewDON(t), handlermocks.NewDON(t)}} - _, _, err := buildShardEndpoints(shardedDONs, connMgrs) - require.Error(t, err) - require.Contains(t, err.Error(), "disjoint") - }) - - t.Run("allMembers unions all shards", func(t *testing.T) { - t.Parallel() + dons, err := handlers.NewShardedDONs(shardedDONs, [][]handlers.DON{{don0, don1}}) + require.NoError(t, err) - eps := []*shardEndpoint{ - {members: []config.NodeConfig{node("n1"), node("n2")}}, - {members: []config.NodeConfig{node("n3")}}, - } - members := allMembers(eps) - require.Len(t, members, 3) + eps, _, err := dons.BuildShardEndpoints() + require.NoError(t, err) + require.Len(t, handlers.AllMembers(eps), 3) }) } diff --git a/core/services/llo/delegate.go b/core/services/llo/delegate.go index 14daafc4f53..fdb78c89df0 100644 --- a/core/services/llo/delegate.go +++ b/core/services/llo/delegate.go @@ -22,6 +22,7 @@ import ( llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" llodatasource "github.com/smartcontractkit/chainlink-data-streams/llo/datasource" llov31 "github.com/smartcontractkit/chainlink-data-streams/llo/dev/v31" + lloconfig "github.com/smartcontractkit/chainlink-data-streams/llo/pluginconfig" lloprotocol "github.com/smartcontractkit/chainlink-data-streams/llo/protocol" "github.com/smartcontractkit/chainlink-data-streams/llo/retirement" "github.com/smartcontractkit/chainlink-data-streams/llo/transmitter" @@ -95,14 +96,17 @@ type DelegateConfig struct { LocalConfig ocr2types.LocalConfig NewOCR3DB func(pluginID int32) ocr3types.Database - // OCR3.1 (only required when OCR31 is true; see chainlink-data-streams - // llo/config.PluginConfig.OCRVersion) - OCR31 bool + // V31 runs the job on the v31 plugin, on libocr OCR3.1 (see + // chainlink-data-streams llo/pluginconfig.PluginConfig.PluginVersion) + V31 bool + // V31Config carries the v31 plugin knobs from the job's plugin config. Only + // read when V31 is true; zero fields fall through to the plugin defaults. + V31Config lloconfig.V31Config // BinaryNetworkEndpoint2Factory is the OCR3.1 ("2") network endpoint factory - // (peerWrapper.Peer3_1). Required when OCR31 is true. + // (peerWrapper.Peer3_1). Required when V31 is true. BinaryNetworkEndpoint2Factory ocr2types.BinaryNetworkEndpoint2Factory // KeyValueDatabaseFactory provides the replicated per-configDigest key-value - // store the OCR3.1 protocol requires. Required when OCR31 is true. + // store the OCR3.1 protocol requires. Required when V31 is true. KeyValueDatabaseFactory ocr3_1types.KeyValueDatabaseFactory } @@ -123,7 +127,7 @@ func NewDelegate(cfg DelegateConfig) (job.ServiceCtx, error) { if cfg.ShouldRetireCache == nil { return nil, errors.New("ShouldRetireCache must not be nil") } - if cfg.OCR31 { + if cfg.V31 { if cfg.KeyValueDatabaseFactory == nil { return nil, errors.New("KeyValueDatabaseFactory must not be nil when running OCR3.1") } @@ -190,7 +194,7 @@ func (d *delegate) Start(ctx context.Context) error { var oracle ocr2plus.Oracle var err error - if d.cfg.OCR31 { + if d.cfg.V31 { oracle, err = d.newOracleV31(i, configTracker, lggr, ocrLogger, psrrc) } else { oracle, err = d.newOracleV30(i, configTracker, lggr, ocrLogger, psrrc) @@ -248,25 +252,38 @@ func (d *delegate) newOracleV30(i int, configTracker ocr2types.ContractConfigTra }) } +// v31FactoryParams assembles the v31 plugin factory params, mapping the job's +// V31Config knobs onto it. Knobs left at zero are forwarded as zero, which the +// factory reads as "apply the plugin default". +func (d *delegate) v31FactoryParams(lggr logger.Logger, psrrc lloprotocol.PredecessorRetirementReportCache) llov31.PluginFactoryParams { + return llov31.PluginFactoryParams{ + VerboseLogging: d.cfg.ReportingPluginConfig.VerboseLogging || d.cfg.V31Config.VerboseLogging, + PredecessorRetirementReportCache: psrrc, + ShouldRetireCache: d.src, + RetirementReportCodec: d.cfg.RetirementReportCodec, + ChannelDefinitionCache: d.cfg.ChannelDefinitionCache, + DataSource: d.ds, + Logger: logger.Named(lggr, "ReportingPlugin"), + OnchainConfigCodec: lloprotocol.EVMOnchainConfigCodec{}, + ReportCodecs: d.reportCodecs, + OutcomeTelemetryCh: d.telem.GetOutcomeTelemetryCh(), + ReportTelemetryCh: d.telem.GetReportTelemetryCh(), + DonID: d.cfg.DonID, + MaxSnapshotRounds: d.cfg.V31Config.MaxSnapshotRounds, + BlobLifetimeRounds: d.cfg.V31Config.BlobLifetimeRounds, + MaxDurationBlobObservation: d.cfg.V31Config.MaxDurationBlobObservation.Duration(), + BlobInFlightWaitFactor: d.cfg.V31Config.BlobInFlightWaitFactor, + MaxBlobSnapshotAge: d.cfg.V31Config.MaxBlobSnapshotAge.Duration(), + MaxRoundPeriod: d.cfg.V31Config.MaxRoundPeriod.Duration(), + } +} + // newOracleV31 builds an OCR3.1 oracle running the llo/v31 reporting plugin. It // differs from v30 by the OCR3.1 oracle args (OCR3_1OracleArgs2), the "2" // network endpoint factory, and the required replicated KeyValueDatabaseFactory. func (d *delegate) newOracleV31(i int, configTracker ocr2types.ContractConfigTracker, lggr logger.Logger, ocrLogger ocrcommontypes.Logger, psrrc lloprotocol.PredecessorRetirementReportCache) (ocr2plus.Oracle, error) { factory := promwrapper31.NewReportingPluginFactory( - llov31.NewPluginFactory(llov31.PluginFactoryParams{ - Config: llov31.Config{VerboseLogging: d.cfg.ReportingPluginConfig.VerboseLogging}, - PredecessorRetirementReportCache: psrrc, - ShouldRetireCache: d.src, - RetirementReportCodec: d.cfg.RetirementReportCodec, - ChannelDefinitionCache: d.cfg.ChannelDefinitionCache, - DataSource: d.ds, - Logger: logger.Named(lggr, "ReportingPlugin"), - OnchainConfigCodec: lloprotocol.EVMOnchainConfigCodec{}, - ReportCodecs: d.reportCodecs, - OutcomeTelemetryCh: d.telem.GetOutcomeTelemetryCh(), - ReportTelemetryCh: d.telem.GetReportTelemetryCh(), - DonID: d.cfg.DonID, - }), + llov31.NewPluginFactory(d.v31FactoryParams(lggr, psrrc)), lggr, "", d.cfg.ChainID, diff --git a/core/services/llo/delegate_test.go b/core/services/llo/delegate_test.go new file mode 100644 index 00000000000..c54bd158820 --- /dev/null +++ b/core/services/llo/delegate_test.go @@ -0,0 +1,96 @@ +package llo + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + lloconfig "github.com/smartcontractkit/chainlink-data-streams/llo/pluginconfig" + llov30 "github.com/smartcontractkit/chainlink-data-streams/llo/v30" + "github.com/smartcontractkit/chainlink/v2/core/services/llo/telem" +) + +// newTestDelegate returns a delegate carrying only what v31FactoryParams reads. +func newTestDelegate(cfg DelegateConfig) *delegate { + return &delegate{cfg: cfg, telem: telem.NewTelemeterService(telem.TelemeterParams{})} +} + +func Test_delegate_v31FactoryParams(t *testing.T) { + t.Parallel() + + t.Run("forwards the V31Config knobs", func(t *testing.T) { + t.Parallel() + + params := newTestDelegate(DelegateConfig{ + DonID: 42, + V31Config: lloconfig.V31Config{ + MaxSnapshotRounds: 7, + BlobLifetimeRounds: 11, + MaxDurationBlobObservation: lloconfig.Duration(3 * time.Second), + BlobInFlightWaitFactor: 5, + MaxBlobSnapshotAge: lloconfig.Duration(9 * time.Second), + MaxRoundPeriod: lloconfig.Duration(time.Minute), + }, + }).v31FactoryParams(logger.Test(t), nil) + + assert.Equal(t, uint64(7), params.MaxSnapshotRounds) + assert.Equal(t, uint64(11), params.BlobLifetimeRounds) + assert.Equal(t, 3*time.Second, params.MaxDurationBlobObservation) + assert.Equal(t, uint64(5), params.BlobInFlightWaitFactor) + assert.Equal(t, 9*time.Second, params.MaxBlobSnapshotAge) + assert.Equal(t, time.Minute, params.MaxRoundPeriod) + assert.Equal(t, uint32(42), params.DonID) + }) + + t.Run("leaves unset knobs zero so the plugin applies its defaults", func(t *testing.T) { + t.Parallel() + + params := newTestDelegate(DelegateConfig{}).v31FactoryParams(logger.Test(t), nil) + + assert.Zero(t, params.MaxSnapshotRounds) + assert.Zero(t, params.BlobLifetimeRounds) + assert.Zero(t, params.MaxDurationBlobObservation) + assert.Zero(t, params.BlobInFlightWaitFactor) + assert.Zero(t, params.MaxBlobSnapshotAge) + assert.Zero(t, params.MaxRoundPeriod) + }) + + t.Run("forwards a negative MaxBlobSnapshotAge, which disables the age check", func(t *testing.T) { + t.Parallel() + + params := newTestDelegate(DelegateConfig{ + V31Config: lloconfig.V31Config{MaxBlobSnapshotAge: lloconfig.Duration(-time.Second)}, + }).v31FactoryParams(logger.Test(t), nil) + + assert.Equal(t, -time.Second, params.MaxBlobSnapshotAge) + }) + + t.Run("enables VerboseLogging from either the node or the job knob", func(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + node bool + job bool + want bool + }{ + {name: "neither", node: false, job: false, want: false}, + {name: "node only", node: true, job: false, want: true}, + {name: "job only", node: false, job: true, want: true}, + {name: "both", node: true, job: true, want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + params := newTestDelegate(DelegateConfig{ + ReportingPluginConfig: llov30.Config{VerboseLogging: tc.node}, + V31Config: lloconfig.V31Config{VerboseLogging: tc.job}, + }).v31FactoryParams(logger.Test(t), nil) + + assert.Equal(t, tc.want, params.VerboseLogging) + }) + } + }) +} diff --git a/core/services/llo/observation/data_source_test.go b/core/services/llo/observation/data_source_test.go index c565aa11cfc..1dc2b2db5a5 100644 --- a/core/services/llo/observation/data_source_test.go +++ b/core/services/llo/observation/data_source_test.go @@ -35,6 +35,7 @@ import ( ) type mockPipeline struct { + mu sync.Mutex run *pipeline.Run trrs pipeline.TaskRunResults err error @@ -46,9 +47,19 @@ type mockPipeline struct { func (m *mockPipeline) Run(ctx context.Context) (*pipeline.Run, pipeline.TaskRunResults, error) { m.runCount.Add(1) + m.mu.Lock() + defer m.mu.Unlock() return m.run, m.trrs, m.err } +// setResults swaps the results returned by Run. Keeping one pipeline pointer lets a test +// change generations without a concurrent observation round mixing results from two pointers. +func (m *mockPipeline) setResults(run *pipeline.Run, trrs pipeline.TaskRunResults, err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.run, m.trrs, m.err = run, trrs, err +} + func (m *mockPipeline) StreamIDs() []streams.StreamID { return m.streamIDs } @@ -593,13 +604,16 @@ func Test_DataSource(t *testing.T) { // Explicitly abort Cycle 1's background observation task before mutating pipelines cancel() - // Fix the pipeline with distinct values so we can verify generation + // Ignore the writes recorded so far; only cycle 2 writes are under test + mc.mu.Lock() + cycle2Start := len(mc.addCalls) + mc.mu.Unlock() + + // Fix the results with distinct values so we can verify generation. Swap them on the + // same pipeline pointer: replacing the registry entries would let a concurrent + // observation round read stream 1 from the old pipeline and streams 2, 3 from the new one. fixedPipeline := makePipelineWithMultipleStreamResults(sids, []any{decimal.NewFromFloat(111.0), decimal.NewFromFloat(222.0), decimal.NewFromFloat(333.0)}) - reg.mu.Lock() - reg.pipelines[1] = fixedPipeline - reg.pipelines[2] = fixedPipeline - reg.pipelines[3] = fixedPipeline - reg.mu.Unlock() + partialPipeline.setResults(fixedPipeline.run, fixedPipeline.trrs, fixedPipeline.err) time.Sleep(observationTimeout * 3) @@ -622,7 +636,7 @@ func Test_DataSource(t *testing.T) { defer mc.mu.Unlock() foundAtomicWrite := false - for _, call := range mc.addCalls { + for _, call := range mc.addCalls[cycle2Start:] { v1, has1 := call.values[llotypes.StreamID(1)] v2, has2 := call.values[llotypes.StreamID(2)] v3, has3 := call.values[llotypes.StreamID(3)] diff --git a/core/services/ocr2/delegate.go b/core/services/ocr2/delegate.go index b2955e4b89a..82f6ee4e327 100644 --- a/core/services/ocr2/delegate.go +++ b/core/services/ocr2/delegate.go @@ -1730,13 +1730,14 @@ func (d *Delegate) newServicesLLO( return NewDB(d.ds, spec.ID, pluginID, lggr) }, - OCR31: pluginCfg.IsOCR31(), + V31: pluginCfg.IsV31(), + V31Config: pluginCfg.V31, } - // OCR3.1 (llo/v31) additionally requires the "2" network endpoint factory and + // The v31 plugin additionally requires the "2" network endpoint factory and // a persistent replicated key-value store, wired here the same way the vault // and DKG OCR3.1 plugins are (pebble under OCR2().KeyValueStoreRootDir()). - if pluginCfg.IsOCR31() { + if pluginCfg.IsV31() { fullPath := filepath.Join(d.cfg.OCR2().KeyValueStoreRootDir(), jb.ExternalJobID.String()) if err = utils.EnsureDirAndMaxPerms(fullPath, os.FileMode(0o700)); err != nil { return nil, fmt.Errorf("failed to create LLO key value store directory: %w", err) diff --git a/core/services/ocr2/plugins/llo/bench/blob_failure_test.go b/core/services/ocr2/plugins/llo/bench/blob_failure_test.go new file mode 100644 index 00000000000..9f981e86370 --- /dev/null +++ b/core/services/ocr2/plugins/llo/bench/blob_failure_test.go @@ -0,0 +1,249 @@ +package bench + +import ( + "context" + "errors" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3_1types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + llov31 "github.com/smartcontractkit/chainlink-data-streams/llo/dev/v31" + "github.com/smartcontractkit/chainlink-data-streams/llo/dev/v31/llotest" +) + +// The v31 blob path carries every stream value: the pump broadcasts a blob of +// observed values, and each round fetches the blobs the observations reference. +// These tests drive the plugin directly (as TestParity does) because neither +// end of that path can be made to fail from the node-level integration tests: +// the blob knobs are not plumbed through the LLO job spec, and the real +// transport does not drop blobs on demand. + +const ( + blobFailureN = 4 + blobFailureF = 1 +) + +// blobFailureWorkload is deliberately tiny: these tests assert on whether +// reports appear at all, not on throughput. +var blobFailureWorkload = workload{numChannels: 2, streamsPerChannel: 1} + +// fetchController wraps the in-memory broadcaster/fetcher so blob fetches can +// be failed on demand, modeling a blob that expired or that no peer served +// before the fetch deadline. +type fetchController struct { + *llotest.BlobBroadcastFetcher + failing atomic.Bool + attempts atomic.Uint64 +} + +func newFetchController() *fetchController { + return &fetchController{BlobBroadcastFetcher: llotest.NewBlobBroadcastFetcher()} +} + +func (f *fetchController) FetchBlob(ctx context.Context, handle ocr3_1types.BlobHandle) ([]byte, error) { + f.attempts.Add(1) + if f.failing.Load() { + return nil, errors.New("blob unavailable") + } + return f.BlobBroadcastFetcher.FetchBlob(ctx, handle) +} + +var _ ocr3_1types.BlobBroadcastFetcher = &fetchController{} + +// newV31 builds a v31 plugin over the given fetcher, applying opts to the +// factory params. +func newV31(tb testing.TB, defs llotypes.ChannelDefinitions, bbf ocr3_1types.BlobBroadcastFetcher, opts ...func(*llov31.PluginFactoryParams)) (ocr3_1types.ReportingPlugin[llotypes.ReportInfo], ocr3_1types.KeyValueDatabase) { + tb.Helper() + p, _, err := newV31Factory(defs, opts...).NewReportingPlugin(context.Background(), pluginConfig(blobFailureN, blobFailureF), bbf) + require.NoError(tb, err) + return p, newKVDB(tb) +} + +// driveRounds drives rounds from startSeq and returns the first seqNr that +// produced reports for every channel, or 0 if none did within the budget. It +// waits for the blob pump between rounds exactly as the benchmark drivers do. +func driveRounds(tb testing.TB, p ocr3_1types.ReportingPlugin[llotypes.ReportInfo], db ocr3_1types.KeyValueDatabase, bbf ocr3_1types.BlobBroadcastFetcher, pump *llotest.BlobBroadcastFetcher, startSeq uint64, rounds int, channels int) (reportingSeq uint64) { + tb.Helper() + for i := range rounds { + seqNr := startSeq + uint64(i) + before := pump.Broadcasts() + reports, _ := v31Round(tb, p, db, bbf, seqNr, blobFailureN) + if seqNr > 1 { + waitForPump(tb, pump, before) + } + if len(reports) >= channels { + return seqNr + } + } + return 0 +} + +// TestBlobBroadcastFailure covers the broadcast end of the blob path: while the +// pump cannot broadcast, no stream values reach the round, so no channel is +// reportable; the protocol must not fail, and reporting must resume once +// broadcasting works again. +func TestBlobBroadcastFailure(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + t.Parallel() + + defs, _ := blobFailureWorkload.channelDefinitions() + bbf := llotest.NewBlobBroadcastFetcher() + bbf.SetBroadcastError(errors.New("broadcast unavailable")) + p, db := newV31(t, defs, bbf) + + budget := int(warmupRounds(blobFailureWorkload.numChannels)) //nolint:gosec // G115: small round budget + seq := driveRounds(t, p, db, bbf, bbf, 1, budget, blobFailureWorkload.numChannels) + require.Zero(t, seq, "no channel should be reportable while blobs cannot be broadcast") + require.Positive(t, bbf.Broadcasts(), "the pump should have attempted to broadcast") + + bbf.SetBroadcastError(nil) + seq = driveRounds(t, p, db, bbf, bbf, uint64(budget+1), budget, blobFailureWorkload.numChannels) + require.NotZero(t, seq, "reporting should resume once blobs can be broadcast again") +} + +// v31RoundErr drives one v31 round like v31Round, but returns the +// StateTransition error instead of failing the test, so the abort path can be +// asserted on. On error nothing is committed, mirroring libocr discarding the +// transaction and retrying the same sequence number. +func v31RoundErr(tb testing.TB, p ocr3_1types.ReportingPlugin[llotypes.ReportInfo], db ocr3_1types.KeyValueDatabase, bbf ocr3_1types.BlobBroadcastFetcher, seqNr uint64) ([]ocr3types.ReportPlus[llotypes.ReportInfo], error) { + tb.Helper() + ctx := context.Background() + + var obs []byte + if seqNr > 1 { + rtx, err := db.NewReadTransaction() + require.NoError(tb, err) + obs, err = p.Observation(ctx, seqNr, ocrtypes.AttributedQuery{}, rtx, bbf) + rtx.Discard() + require.NoError(tb, err) + } + aos := bootOrReplicate(obs, blobFailureN, seqNr) + + wtx, err := db.NewReadWriteTransaction() + require.NoError(tb, err) + prec, err := p.StateTransition(ctx, seqNr, ocrtypes.AttributedQuery{}, aos, wtx, bbf) + if err != nil { + wtx.Discard() + return nil, err + } + require.NoError(tb, wtx.Commit()) + + reports, err := p.Reports(ctx, seqNr, prec) + require.NoError(tb, err) + return reports, nil +} + +// TestBlobFetchFailure covers the fetch end: the blobs are broadcast, but the +// round cannot retrieve them (expired, or no peer served them in time). A +// fetch failure is node-local, so dropping the observation would make the state +// transition non-deterministic across oracles; the round must abort instead, +// and reporting must resume once fetches succeed. +func TestBlobFetchFailure(t *testing.T) { + if testing.Short() { + t.Skip("too slow for testing.Short") + } + t.Parallel() + + defs, _ := blobFailureWorkload.channelDefinitions() + fc := newFetchController() + fc.failing.Store(true) + p, db := newV31(t, defs, fc) + + budget := int(warmupRounds(blobFailureWorkload.numChannels)) //nolint:gosec // G115: small round budget + + // Drive until a round references a blob and therefore aborts. Earlier + // rounds only carry channel definitions, which travel inline. + var seqNr uint64 = 1 + var abortErr error + for range budget { + reports, err := v31RoundErr(t, p, db, fc, seqNr) + if err != nil { + abortErr = err + break + } + require.Empty(t, reports, "no channel should be reportable while blobs cannot be fetched") + before := fc.Broadcasts() + seqNr++ + waitForPump(t, fc.BlobBroadcastFetcher, before) + } + require.ErrorContains(t, abortErr, "failed to fetch blob for observation", + "a round referencing an unfetchable blob should abort") + require.Positive(t, fc.attempts.Load(), "the round should have attempted to fetch the blobs it referenced") + + // The aborted round committed nothing, so the same sequence number is + // retried, which is what libocr does. + fc.failing.Store(false) + seq := driveRounds(t, p, db, fc, fc.BlobBroadcastFetcher, seqNr, budget, blobFailureWorkload.numChannels) + require.NotZero(t, seq, "reporting should resume once blobs can be fetched again") +} + +// TestBlobRoundBounds covers the blob round knobs the factory validates: a blob +// lifetime that does not outlive the snapshots referencing it is rejected, as +// is one past the cap, while the tightest valid pairing still reports. +func TestBlobRoundBounds(t *testing.T) { + t.Parallel() + + // The tightest valid pairing: a snapshot gathered for one sequence number + // is consumed by a later one, so the blob must stay fetchable for + // BlobFetchMarginRounds past MaxSnapshotRounds. + const tightSnapshotRounds = 2 + const tightLifetimeRounds = tightSnapshotRounds + llov31.BlobFetchMarginRounds - 1 + + tcs := []struct { + name string + maxSnapshotRounds uint64 + blobLifetime uint64 + wantErr string + }{ + { + name: "lifetime leaving no fetch margin is rejected", + maxSnapshotRounds: 4, + blobLifetime: 4, + wantErr: "leaves less than", + }, + { + name: "lifetime past the cap is rejected", + blobLifetime: llov31.MaxBlobLifetimeRounds + 1, + wantErr: "exceeds MaxBlobLifetimeRounds", + }, + { + name: "tightest valid pairing reports", + maxSnapshotRounds: tightSnapshotRounds, + blobLifetime: tightLifetimeRounds, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + defs, _ := blobFailureWorkload.channelDefinitions() + opt := func(params *llov31.PluginFactoryParams) { + params.MaxSnapshotRounds = tc.maxSnapshotRounds + params.BlobLifetimeRounds = tc.blobLifetime + } + bbf := llotest.NewBlobBroadcastFetcher() + + if tc.wantErr != "" { + _, _, err := newV31Factory(defs, opt).NewReportingPlugin(context.Background(), pluginConfig(blobFailureN, blobFailureF), bbf) + require.ErrorContains(t, err, tc.wantErr) + return + } + + if testing.Short() { + t.Skip("too slow for testing.Short") + } + p, db := newV31(t, defs, bbf, opt) + budget := int(warmupRounds(blobFailureWorkload.numChannels)) //nolint:gosec // G115: small round budget + seq := driveRounds(t, p, db, bbf, bbf, 1, budget, blobFailureWorkload.numChannels) + require.NotZero(t, seq, "the tightest valid blob round bounds should still report") + }) + } +} diff --git a/core/services/ocr2/plugins/llo/bench/harness_test.go b/core/services/ocr2/plugins/llo/bench/harness_test.go index 54022c3b16a..d4a2e049c21 100644 --- a/core/services/ocr2/plugins/llo/bench/harness_test.go +++ b/core/services/ocr2/plugins/llo/bench/harness_test.go @@ -183,10 +183,14 @@ func reportCodecs() map[llotypes.ReportFormat]lloprotocol.ReportCodec { // to whole seconds, which would prevent reporting within a single wall-clock // second and diverge from v31). The 1ns interval effectively reports every // round while keeping both plugins on identical reportability rules. -func benchOffchainConfig() []byte { +// aggregationFaultTolerance is set to consensus F, the maximum the v31 factory +// accepts, so both plugins aggregate over the same observation set. +func benchOffchainConfig(f int) []byte { + aft := uint32(f) //nolint:gosec // bench F is a small positive constant b, err := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: 1, + AggregationFaultTolerance: &aft, }.Encode() if err != nil { panic(err) @@ -200,14 +204,14 @@ func pluginConfig(n, f int) ocr3types.ReportingPluginConfig { N: n, F: f, MaxDurationObservation: maxDurationObservation, - OffchainConfig: benchOffchainConfig(), + OffchainConfig: benchOffchainConfig(f), } } func buildV30(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) ocr3types.ReportingPlugin[llotypes.ReportInfo] { tb.Helper() factory := llov30.NewPluginFactory(llov30.PluginFactoryParams{ - Config: llov30.Config{VerboseLogging: false}, + VerboseLogging: false, ShouldRetireCache: mockShouldRetireCache{}, RetirementReportCodec: lloprotocol.StandardRetirementReportCodec{}, ChannelDefinitionCache: &mockChannelDefinitionCache{defs: defs}, @@ -221,10 +225,11 @@ func buildV30(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) ocr3typ return p } -func buildV31(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) (ocr3_1types.ReportingPlugin[llotypes.ReportInfo], ocr3_1types.KeyValueDatabase, *llotest.BlobBroadcastFetcher) { - tb.Helper() - factory := llov31.NewPluginFactory(llov31.PluginFactoryParams{ - Config: llov31.Config{VerboseLogging: false}, +// newV31Factory builds the v31 plugin factory over the given definitions. opts +// mutate the factory params, e.g. to override the blob pump knobs. +func newV31Factory(defs llotypes.ChannelDefinitions, opts ...func(*llov31.PluginFactoryParams)) *llov31.PluginFactory { + params := llov31.PluginFactoryParams{ + VerboseLogging: false, ShouldRetireCache: mockShouldRetireCache{}, RetirementReportCodec: lloprotocol.StandardRetirementReportCodec{}, ChannelDefinitionCache: &mockChannelDefinitionCache{defs: defs}, @@ -232,26 +237,38 @@ func buildV31(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) (ocr3_1 Logger: logger.Nop(), OnchainConfigCodec: mockOnchainConfigCodec{}, ReportCodecs: reportCodecs(), - }) + } + for _, opt := range opts { + opt(¶ms) + } + return llov31.NewPluginFactory(params) +} + +// newKVDB returns libocr's in-memory KeyValueDatabase (the same helper v31's +// integration tests use): a btree behind the production KeyValueDatabaseFactory +// interface, whose Commit applies to memory with no WAL/fsync. This isolates +// the plugin's CPU/allocation cost from storage-engine cost, so the v31 numbers +// are directly comparable to v30's in-memory Outcome blob. +func newKVDB(tb testing.TB) ocr3_1types.KeyValueDatabase { + tb.Helper() + dbFactory := memkvdb.NewStatelessInMemoryKeyValueDatabaseFactory() + db, err := dbFactory.NewKeyValueDatabase(benchConfigDigest) + require.NoError(tb, err) + tb.Cleanup(func() { _ = db.Close() }) + return db +} + +func buildV31(tb testing.TB, defs llotypes.ChannelDefinitions, n, f int) (ocr3_1types.ReportingPlugin[llotypes.ReportInfo], ocr3_1types.KeyValueDatabase, *llotest.BlobBroadcastFetcher) { + tb.Helper() // v31 disseminates stream values exclusively through blobs, so the plugin // needs a BlobBroadcastFetcher both at construction (the blob pump // broadcasts through it) and on every Observation/StateTransition call (the // round decodes observations by fetching the blobs they reference). A nil // fetcher yields a plugin whose observations never carry stream values. bbf := llotest.NewBlobBroadcastFetcher() - p, _, err := factory.NewReportingPlugin(context.Background(), pluginConfig(n, f), bbf) + p, _, err := newV31Factory(defs).NewReportingPlugin(context.Background(), pluginConfig(n, f), bbf) require.NoError(tb, err) - - // libocr's in-memory KeyValueDatabase (the same helper v31's integration - // tests use): a btree behind the production KeyValueDatabaseFactory - // interface, whose Commit applies to memory with no WAL/fsync. This isolates - // the plugin's CPU/allocation cost from storage-engine cost, so the v31 - // numbers are directly comparable to v30's in-memory Outcome blob. - dbFactory := memkvdb.NewStatelessInMemoryKeyValueDatabaseFactory() - db, err := dbFactory.NewKeyValueDatabase(benchConfigDigest) - require.NoError(tb, err) - tb.Cleanup(func() { _ = db.Close() }) - return p, db, bbf + return p, newKVDB(tb), bbf } // --------------------------------------------------------------------------- diff --git a/core/services/ocr2/plugins/llo/helpers_test.go b/core/services/ocr2/plugins/llo/helpers_test.go index 09fcc3234cf..eaaac68ec39 100644 --- a/core/services/ocr2/plugins/llo/helpers_test.go +++ b/core/services/ocr2/plugins/llo/helpers_test.go @@ -136,12 +136,13 @@ func (node *Node) DeleteJob(t *testing.T, id int32) { require.NoError(t, err) } -func (node *Node) AddLLOJob(t *testing.T, spec string) { +func (node *Node) AddLLOJob(t *testing.T, spec string) (id int32) { c := node.App.GetConfig() job, err := validate.ValidatedOracleSpecToml(t.Context(), c.OCR2(), c.Insecure(), spec, nil) require.NoError(t, err) err = node.App.AddJobV2(t.Context(), &job) require.NoError(t, err) + return job.ID } func (node *Node) AddBootstrapJob(t *testing.T, spec string) { @@ -159,6 +160,23 @@ func setupNode( csaKey csakey.KeyV2, f func(*chainlink.Config), ) (app chainlink.Application, peerID string, clientPubKey credentials.StaticSizedPublicKey, ocr2kb ocr2key.KeyBundle, observedLogs *observer.ObservedLogs) { + app, peerID, clientPubKey, ocr2kb, observedLogs, _ = setupRestartableNode(t, port, dbName, backend, csaKey, f) + return app, peerID, clientPubKey, ocr2kb, observedLogs +} + +// setupRestartableNode is setupNode plus a restart hook. restart stops the +// application, runs whileDown (if any), then starts a new one on the same +// config, database, keys and key-value store root, which is a process restart +// with every piece of persisted state left in place: jobs, and the OCR3.1 +// (llo/v31) pebble store. +func setupRestartableNode( + t *testing.T, + port int, + dbName string, + backend evmtypes.Backend, + csaKey csakey.KeyV2, + f func(*chainlink.Config), +) (app chainlink.Application, peerID string, clientPubKey credentials.StaticSizedPublicKey, ocr2kb ocr2key.KeyBundle, observedLogs *observer.ObservedLogs, restart func(t *testing.T, whileDown func()) chainlink.Application) { k := big.NewInt(int64(port)) // keys unique to port p2pKey := p2pkey.MustNewV2XXXTestingOnly(k) rdr := keystest.NewRandReaderFromSeed(int64(port)) @@ -221,19 +239,32 @@ func setupNode( }) lggr, observedLogs := logger.TestLoggerObserved(t, config.Log().Level()) - if backend != nil { - app = cltest.NewApplicationWithConfigV2OnSimulatedBlockchain(t, config, backend, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) - } else { - app = cltest.NewApplicationWithConfig(t, config, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) + // newApp builds and starts an application over the config above. Start + // registers its own cleanup, which tolerates an application this test + // already stopped. + newApp := func(t *testing.T) chainlink.Application { + var a chainlink.Application + if backend != nil { + a = cltest.NewApplicationWithConfigV2OnSimulatedBlockchain(t, config, backend, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) + } else { + a = cltest.NewApplicationWithConfig(t, config, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) + } + require.NoError(t, a.Start(t.Context())) + return a } - err := app.Start(t.Context()) - require.NoError(t, err) - t.Cleanup(func() { - assert.NoError(t, app.Stop()) - }) + app = newApp(t) + current := app + restart = func(t *testing.T, whileDown func()) chainlink.Application { + require.NoError(t, current.Stop()) + if whileDown != nil { + whileDown() + } + current = newApp(t) + return current + } - return app, p2pKey.PeerID().Raw(), csaKey.StaticSizedPublicKey(), ocr2kb, observedLogs + return app, p2pKey.PeerID().Raw(), csaKey.StaticSizedPublicKey(), ocr2kb, observedLogs, restart } // receiveWithTimeout receives from the packet channel with a timeout. @@ -368,8 +399,8 @@ func addLLOJob( pluginConfig, relayType, relayConfig string, -) { - node.AddLLOJob(t, fmt.Sprintf(` +) (id int32) { + return node.AddLLOJob(t, fmt.Sprintf(` type = "offchainreporting2" schemaVersion = 1 name = "%s" diff --git a/core/services/ocr2/plugins/llo/history_backfill_integration_test.go b/core/services/ocr2/plugins/llo/history_backfill_integration_test.go index a5b150e5178..c0dc999bd4c 100644 --- a/core/services/ocr2/plugins/llo/history_backfill_integration_test.go +++ b/core/services/ocr2/plugins/llo/history_backfill_integration_test.go @@ -104,19 +104,19 @@ func quoteBackfillString(benchmark float64) string { func TestIntegration_LLO_history_backfill(t *testing.T) { t.Parallel() - for _, ocr31 := range []bool{false, true} { + for _, v31 := range []bool{false, true} { name := "OCR3.0/v30" - if ocr31 { + if v31 { name = "OCR3.1/v31" } t.Run(name, func(t *testing.T) { t.Parallel() - testIntegrationLLOHistoryBackfill(t, ocr31) + testIntegrationLLOHistoryBackfill(t, v31) }) } } -func testIntegrationLLOHistoryBackfill(t *testing.T, ocr31 bool) { +func testIntegrationLLOHistoryBackfill(t *testing.T, v31 bool) { const ( salt = 600 donID = uint32(776655) @@ -134,6 +134,7 @@ func testIntegrationLLOHistoryBackfill(t *testing.T, ocr31 bool) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: true, } @@ -178,8 +179,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } nativeStrm := Stream{ @@ -223,8 +224,8 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi backend.Commit() productionConfigOpts := []OCRConfigOption{WithOracles(oracles), WithOffchainConfig(offchainConfig)} - if ocr31 { - productionConfigOpts = append(productionConfigOpts, WithOCR31()) + if v31 { + productionConfigOpts = append(productionConfigOpts, WithV31()) } setProductionConfig( t, donID, steve, backend, configurator, configuratorAddress, nodes, diff --git a/core/services/ocr2/plugins/llo/integration_test.go b/core/services/ocr2/plugins/llo/integration_test.go index 82bf18a025c..a1255e055da 100644 --- a/core/services/ocr2/plugins/llo/integration_test.go +++ b/core/services/ocr2/plugins/llo/integration_test.go @@ -73,6 +73,10 @@ var ( fNodes = uint8(1) nNodes = 4 // number of nodes (not including bootstrap) reportTimeout = time.Second * 60 + + // v31 requires the contribution floor to be set explicitly. Match consensus + // F so aggregation uses the full observation set. + aggregationFaultTolerance = func() *uint32 { v := uint32(fNodes); return &v }() ) func setupBlockchain(t *testing.T, adders ...*bind.TransactOpts) ( @@ -227,11 +231,11 @@ type OCRConfig struct { F int OnchainConfig []byte - // ocr31 selects the OCR3.1 (llo/v31) config format when true. It changes the + // v31 selects the OCR3.1 config format the v31 plugin runs on. It changes the // confighelper used by generateConfig (ocr3_1confighelper => offchainConfigVersion // 310) so the on-chain config matches what an OCR3.1 node validates. The // config digest prefix (LLO 0x0009) is identical across OCR3.0/3.1. - ocr31 bool + v31 bool } func makeDefaultOCRConfig() *OCRConfig { @@ -292,10 +296,10 @@ func WithOracles(oracles []confighelper.OracleIdentityExtra) OCRConfigOption { } } -// WithOCR31 switches config generation to the OCR3.1 (llo/v31) format. -func WithOCR31() OCRConfigOption { +// WithV31 switches config generation to the OCR3.1 format the v31 plugin runs on. +func WithV31() OCRConfigOption { return func(cfg *OCRConfig) { - cfg.ocr31 = true + cfg.v31 = true } } @@ -309,7 +313,7 @@ func generateConfig(t *testing.T, opts ...OCRConfigOption) (signers []types.Onch } t.Logf("Using OCR config: %+v\n", cfg) var err error - if cfg.ocr31 { + if cfg.v31 { signers, transmitters, f, outOnchainConfig, offchainConfigVersion, offchainConfig, err = generateOCR31Config(cfg) } else { signers, transmitters, f, outOnchainConfig, offchainConfigVersion, offchainConfig, err = ocr3confighelper.ContractSetConfigArgsForTests( @@ -483,22 +487,54 @@ func promoteStagingConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, backend.Commit() } -func TestIntegration_LLO_evm_premium_legacy(t *testing.T) { - t.Parallel() - offchainConfigs := []lloprotocol.OffchainConfig{ +// offchainConfigCase is a named offchain config for a test matrix. The name is +// spelled out rather than derived from the struct so subtest names stay stable +// (the struct holds a pointer, whose formatted address changes per run). +type offchainConfigCase struct { + name string + cfg lloprotocol.OffchainConfig +} + +// offchainConfigCases is the offchain config matrix shared by the tests that +// cover every report format: both protocol versions, plus observation +// compression. Compression is a v30 codec setting; v31 always frames its blobs +// with zstd and ignores the flag. +func offchainConfigCases() []offchainConfigCase { + return []offchainConfigCase{ + { + name: "protocolVersion=0", + cfg: lloprotocol.OffchainConfig{ + ProtocolVersion: 0, + DefaultMinReportIntervalNanoseconds: 0, + AggregationFaultTolerance: aggregationFaultTolerance, + }, + }, { - ProtocolVersion: 0, - DefaultMinReportIntervalNanoseconds: 0, + name: "protocolVersion=1", + cfg: lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: 1, + AggregationFaultTolerance: aggregationFaultTolerance, + }, }, { - ProtocolVersion: 1, - DefaultMinReportIntervalNanoseconds: 1, + name: "protocolVersion=1/compressedObservations", + cfg: lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: 1, + AggregationFaultTolerance: aggregationFaultTolerance, + EnableObservationCompression: true, + }, }, } - for _, offchainConfig := range offchainConfigs { - t.Run(fmt.Sprintf("offchainConfig=%+v", offchainConfig), func(t *testing.T) { +} + +func TestIntegration_LLO_evm_premium_legacy(t *testing.T) { + t.Parallel() + for _, oc := range offchainConfigCases() { + t.Run(oc.name, func(t *testing.T) { t.Parallel() - testIntegrationLLOEVMPremiumLegacy(t, offchainConfig) + testIntegrationLLOEVMPremiumLegacy(t, oc.cfg) }) } } @@ -725,34 +761,24 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi func TestIntegration_LLO_multi_formats(t *testing.T) { t.Parallel() - offchainConfigs := []lloprotocol.OffchainConfig{ - { - ProtocolVersion: 0, - DefaultMinReportIntervalNanoseconds: 0, - }, - { - ProtocolVersion: 1, - DefaultMinReportIntervalNanoseconds: 1, - }, - } - ocrVersions := []struct { - name string - ocr31 bool + pluginVersions := []struct { + name string + v31 bool }{ {"OCR3.0/v30", false}, {"OCR3.1/v31", true}, } - for _, offchainConfig := range offchainConfigs { - for _, ov := range ocrVersions { - t.Run(fmt.Sprintf("%s/offchainConfig=%+v", ov.name, offchainConfig), func(t *testing.T) { + for _, oc := range offchainConfigCases() { + for _, ov := range pluginVersions { + t.Run(fmt.Sprintf("%s/%s", ov.name, oc.name), func(t *testing.T) { t.Parallel() - testIntegrationLLOMultiFormats(t, offchainConfig, ov.ocr31) + testIntegrationLLOMultiFormats(t, oc.cfg, ov.v31) }) } } } -func testIntegrationLLOMultiFormats(t *testing.T, offchainConfig lloprotocol.OffchainConfig, ocr31 bool) { +func testIntegrationLLOMultiFormats(t *testing.T, offchainConfig lloprotocol.OffchainConfig, v31 bool) { testStartTimeStamp := time.Now() expirationWindow := uint32(3600) @@ -1088,8 +1114,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } bridgeName := "superbridge" @@ -1279,8 +1305,8 @@ dp -> deribit_funding_interval_hours_parse -> deribit_funding_interval_hours_dec // Set config on configurator productionConfigOpts := []OCRConfigOption{WithOracles(oracles), WithOffchainConfig(offchainConfig)} - if ocr31 { - productionConfigOpts = append(productionConfigOpts, WithOCR31()) + if v31 { + productionConfigOpts = append(productionConfigOpts, WithV31()) } digest := setProductionConfig( t, donID, steve, backend, configurator, configuratorAddress, nodes, productionConfigOpts..., @@ -1523,6 +1549,7 @@ func TestIntegration_LLO_stress_test_V1(t *testing.T) { WithOffchainConfig(lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(defaultMinReportInterval), + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: true, }), func(cfg *OCRConfig) { @@ -1759,6 +1786,7 @@ func TestIntegration_LLO_transmit_errors(t *testing.T) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(50 * time.Millisecond), + AggregationFaultTolerance: aggregationFaultTolerance, } clientCSAKeys := make([]csakey.KeyV2, nNodes) @@ -1907,6 +1935,47 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, serverPubKey }) } +// assertRetirementReport checks the attested retirement report every oracle +// persisted for a retired config. The successor instance reads this report to +// learn where the predecessor stopped, so its contents are what make the +// handover gapless: a wrong ValidAfterNanoseconds either replays or skips a +// window of time that no report covers. +// +// truncateToSecond covers protocol version 0, whose timestamps carry only +// second resolution. +func assertRetirementReport(t *testing.T, nodes []Node, digest ocr2types.ConfigDigest, wantProtocolVersion uint32, wantValidAfterNanoseconds map[llotypes.ChannelID]uint64, truncateToSecond bool) { + t.Helper() + + toSecond := func(m map[llotypes.ChannelID]uint64) map[llotypes.ChannelID]uint64 { + if !truncateToSecond { + return m + } + out := make(map[llotypes.ChannelID]uint64, len(m)) + for k, v := range m { + out[k] = v / 1e9 * 1e9 + } + return out + } + + for i, node := range nodes { + var raw []byte + require.Eventually(t, func() bool { + return node.App.GetDB().GetContext(t.Context(), &raw, + `SELECT attested_retirement_report FROM llo_retirement_report_cache WHERE config_digest = $1`, digest[:]) == nil + }, reportTimeout, 100*time.Millisecond, "oracle %d should persist an attested retirement report for the retired config", i) + + var attested lloprotocol.AttestedRetirementReport + require.NoError(t, proto.Unmarshal(raw, &attested)) + assert.NotZero(t, attested.SeqNr, "oracle %d: retirement report should name the sequence number it retired at", i) + assert.GreaterOrEqual(t, len(attested.Sigs), int(fNodes)+1, "oracle %d: an attested retirement report needs at least f+1 signatures", i) + + rr, err := (lloprotocol.StandardRetirementReportCodec{}).Decode(attested.RetirementReport) + require.NoError(t, err) + assert.Equal(t, wantProtocolVersion, rr.ProtocolVersion, "oracle %d", i) + assert.Equal(t, toSecond(wantValidAfterNanoseconds), toSecond(rr.ValidAfterNanoseconds), "oracle %d", i) + } +} + func TestIntegration_LLO_blue_green_lifecycle(t *testing.T) { t.Parallel() @@ -1915,25 +1984,26 @@ func TestIntegration_LLO_blue_green_lifecycle(t *testing.T) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 0, DefaultMinReportIntervalNanoseconds: 0, + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: false, } - for _, ocr31 := range []bool{false, true} { + for _, v31 := range []bool{false, true} { name := "OCR3.0/v30" - if ocr31 { + if v31 { name = "OCR3.1/v31" } t.Run(name, func(t *testing.T) { t.Parallel() - testIntegrationLLOBlueGreenLifecycle(t, offchainConfig, ocr31) + testIntegrationLLOBlueGreenLifecycle(t, offchainConfig, v31) }) } } -func testIntegrationLLOBlueGreenLifecycle(t *testing.T, offchainConfig lloprotocol.OffchainConfig, ocr31 bool) { - // withVersion appends WithOCR31() to config options when running the v31 variant. +func testIntegrationLLOBlueGreenLifecycle(t *testing.T, offchainConfig lloprotocol.OffchainConfig, v31 bool) { + // withVersion appends WithV31() to config options when running the v31 variant. withVersion := func(opts ...OCRConfigOption) []OCRConfigOption { - if ocr31 { - return append(opts, WithOCR31()) + if v31 { + return append(opts, WithV31()) } return opts } @@ -2011,8 +2081,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } addOCRJobsEVMPremiumLegacy(t, streams, serverPubKey, serverURL, configuratorAddress, bootstrapPeerID, bootstrapNodePort, nodes, configStoreAddress, clientPubKeys, pluginConfig, relayType, relayConfig) @@ -2161,6 +2231,11 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi } assert.Less(t, initialPromotedGreenReport.ValidAfterNanoseconds, initialPromotedGreenReport.ObservationTimestampNanoseconds) + + // The retired instance's report is what green handed over from. + assertRetirementReport(t, nodes, blueDigest, offchainConfig.ProtocolVersion, + map[llotypes.ChannelID]uint64{1: finalBlueReport.ObservationTimestampNanoseconds}, + offchainConfig.ProtocolVersion == 0) } // retired instance does not produce reports { @@ -2237,6 +2312,11 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi finalGreenReport := allReports[greenDigest][len(allReports[greenDigest])-1] assert.Less(t, finalGreenReport.ValidAfterNanoseconds, finalGreenReport.ObservationTimestampNanoseconds) assert.Less(t, initialPromotedBlueReport.ValidAfterNanoseconds, initialPromotedBlueReport.ObservationTimestampNanoseconds) + + // Green ran on the config it was staged with, which is the one blue + // retired under; blue's new staging config is not yet in effect. + assertRetirementReport(t, nodes, greenDigest, 0, + map[llotypes.ChannelID]uint64{1: finalGreenReport.ObservationTimestampNanoseconds}, true) } // adding a new channel definition is picked up on the fly { @@ -2282,33 +2362,28 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi assert.Equal(t, "2976.39", r.Values[0].(*lloprotocol.Decimal).String()) } } - t.Run("deleting the jobs turns off oracles and cleans up resources", func(t *testing.T) { - t.Skip("TODO - MERC-3524") - }) - t.Run("adding new jobs again picks up the correct configs", func(t *testing.T) { - t.Skip("TODO - MERC-3524") - }) }) } func TestIntegration_LLO_channel_merging_owners_adders(t *testing.T) { t.Parallel() - for _, ocr31 := range []bool{false, true} { + for _, v31 := range []bool{false, true} { name := "OCR3.0/v30" - if ocr31 { + if v31 { name = "OCR3.1/v31" } t.Run(name, func(t *testing.T) { t.Parallel() - testIntegrationLLOChannelMerging(t, ocr31) + testIntegrationLLOChannelMerging(t, v31) }) } } -func testIntegrationLLOChannelMerging(t *testing.T, ocr31 bool) { +func testIntegrationLLOChannelMerging(t *testing.T, v31 bool) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: true, } @@ -2387,8 +2462,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } // Add stream specs and LLO jobs to all nodes @@ -2410,8 +2485,8 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi // Set initial OCR config mergeConfigOpts := []OCRConfigOption{WithOracles(oracles), WithOffchainConfig(offchainConfig)} - if ocr31 { - mergeConfigOpts = append(mergeConfigOpts, WithOCR31()) + if v31 { + mergeConfigOpts = append(mergeConfigOpts, WithV31()) } digest := setProductionConfig( t, donID, steve, backend, configurator, configuratorAddress, nodes, mergeConfigOpts..., @@ -2818,19 +2893,19 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi // and no longer transmits reports for that channel. func TestIntegration_LLO_tombstone_stops_observations_and_reports(t *testing.T) { t.Parallel() - for _, ocr31 := range []bool{false, true} { + for _, v31 := range []bool{false, true} { name := "OCR3.0/v30" - if ocr31 { + if v31 { name = "OCR3.1/v31" } t.Run(name, func(t *testing.T) { t.Parallel() - testIntegrationLLOTombstone(t, ocr31) + testIntegrationLLOTombstone(t, v31) }) } } -func testIntegrationLLOTombstone(t *testing.T, ocr31 bool) { +func testIntegrationLLOTombstone(t *testing.T, v31 bool) { const ( salt = 500 donID = uint32(777666) @@ -2841,6 +2916,7 @@ func testIntegrationLLOTombstone(t *testing.T, ocr31 bool) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, EnableObservationCompression: true, } @@ -2885,8 +2961,8 @@ lloConfigMode = "bluegreen" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - if ocr31 { - pluginConfig += "\nocrVersion = \"3.1\"" + if v31 { + pluginConfig += "\npluginVersion = \"v31\"" } var streamACalls, streamBCalls atomic.Uint64 @@ -2931,8 +3007,8 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi backend.Commit() tombstoneConfigOpts := []OCRConfigOption{WithOracles(oracles), WithOffchainConfig(offchainConfig)} - if ocr31 { - tombstoneConfigOpts = append(tombstoneConfigOpts, WithOCR31()) + if v31 { + tombstoneConfigOpts = append(tombstoneConfigOpts, WithV31()) } setProductionConfig( t, donID, steve, backend, configurator, configuratorAddress, nodes, @@ -3047,6 +3123,7 @@ func TestIntegration_LLO_bridgeConnManagerHappyPath(t *testing.T) { offchainConfig := lloprotocol.OffchainConfig{ ProtocolVersion: 1, DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, } clientCSAKeys := make([]csakey.KeyV2, nNodes) @@ -3156,9 +3233,17 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi } func setupNodes(t *testing.T, nNodes int, backend evmtypes.Backend, clientCSAKeys []csakey.KeyV2, f func(*chainlink.Config)) (oracles []confighelper.OracleIdentityExtra, nodes []Node) { + oracles, nodes, _ = setupNodesWithRestart(t, nNodes, backend, clientCSAKeys, f) + return oracles, nodes +} + +// setupNodesWithRestart is setupNodes plus a per-node restart hook. See +// setupRestartableNode. +func setupNodesWithRestart(t *testing.T, nNodes int, backend evmtypes.Backend, clientCSAKeys []csakey.KeyV2, f func(*chainlink.Config)) (oracles []confighelper.OracleIdentityExtra, nodes []Node, restarts []func(t *testing.T, whileDown func()) chainlink.Application) { ports := freeport.GetN(t, nNodes) for i := range nNodes { - app, peerID, transmitter, kb, observedLogs := setupNode(t, ports[i], fmt.Sprintf("oracle_streams_%d", i), backend, clientCSAKeys[i], f) + app, peerID, transmitter, kb, observedLogs, restart := setupRestartableNode(t, ports[i], fmt.Sprintf("oracle_streams_%d", i), backend, clientCSAKeys[i], f) + restarts = append(restarts, restart) nodes = append(nodes, Node{ app, transmitter, kb, observedLogs, @@ -3175,7 +3260,7 @@ func setupNodes(t *testing.T, nNodes int, backend evmtypes.Backend, clientCSAKey ConfigEncryptionPublicKey: kb.ConfigEncryptionPublicKey(), }) } - return oracles, nodes + return oracles, nodes, restarts } func newChannelDefinitionsServer(t *testing.T, channelDefinitions llotypes.ChannelDefinitions) (url string, sha [32]byte) { @@ -3234,3 +3319,563 @@ func newSingleABIEncoder(typ string, multiplier *sqlutil.Big) (enc lloevm.ABIEnc } return enc } + +// TestIntegration_LLO_aggregation_fault_tolerance exercises the v31 contribution +// floor. aggregationFaultTolerance sets the fewest contributions an aggregate may +// be built from (2*aft+1), so a stream observed by too few oracles must not +// produce an aggregate and its channel must not report, while a stream observed +// by the whole DON keeps reporting. +// +// With nNodes=4 and F=1: aft=1 sets a floor of 3 contributions, aft=0 a floor of +// 1. A stream whose job exists on only 2 nodes therefore reports under aft=0 and +// is withheld under aft=1. +func TestIntegration_LLO_aggregation_fault_tolerance(t *testing.T) { + t.Parallel() + tcs := []struct { + name string + // aft is the aggregationFaultTolerance set in the offchain config. + aft uint32 + // partialReports is whether the channel whose stream only 2 of the 4 + // oracles observe is expected to produce reports. + partialReports bool + salt int64 + donID uint32 + }{ + {name: "floor of 3 withholds an under-observed stream", aft: 1, partialReports: false, salt: 700, donID: uint32(555111)}, + {name: "floor of 1 admits an under-observed stream", aft: 0, partialReports: true, salt: 750, donID: uint32(555222)}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testIntegrationLLOAggregationFaultTolerance(t, tc.aft, tc.partialReports, tc.salt, tc.donID) + }) + } +} + +func testIntegrationLLOAggregationFaultTolerance(t *testing.T, aft uint32, expectPartialReports bool, salt int64, donID uint32) { + const ( + // streamIDFull has a stream job on every oracle. + streamIDFull = uint32(690) + // streamIDPartial has a stream job on only the first two oracles, so it + // gathers 2 contributions per round. + streamIDPartial = uint32(691) + // nPartialObservers is how many oracles observe streamIDPartial. + nPartialObservers = 2 + + channelFull = uint32(1) + channelPartial = uint32(2) + ) + + offchainConfig := lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: &aft, + } + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + for i := range nNodes { + k := big.NewInt(salt + int64(i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress, _, _, _, _ := setupBlockchain(t) + fromBlock := 1 + + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo_aft", backend, bootstrapCSAKey, nil) + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + packetCh := make(chan *packet, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, serverKey, packetCh) + serverURL := startMercuryServer(t, srv, clientPubKeys) + + oracles, nodes := setupNodes(t, nNodes, backend, clientCSAKeys, func(c *chainlink.Config) { + c.Mercury.Transmitter.Protocol = new(mercurytransmitter.MercuryTransmitterProtocolGRPC) + }) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, fmt.Sprintf("job-aft-%d", donID), relayType, relayConfig) + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d +pluginVersion = "v31"`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + + priceFull := decimal.NewFromFloat(111.1) + pricePartial := decimal.NewFromFloat(222.2) + for i, node := range nodes { + bridgeFull := createSingleDecimalBridge(t, "aft-full", i, priceFull, node.App.BridgeORM()) + addSingleDecimalStreamJob(t, node, streamIDFull, bridgeFull) + if i < nPartialObservers { + bridgePartial := createSingleDecimalBridge(t, "aft-partial", i, pricePartial, node.App.BridgeORM()) + addSingleDecimalStreamJob(t, node, streamIDPartial, bridgePartial) + } + addLLOJob( + t, + node, + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[i], + "aggregation-fault-tolerance-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + channelDefinitions := llotypes.ChannelDefinitions{ + channelFull: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + {StreamID: streamIDFull, Aggregator: llotypes.AggregatorMedian}, + }, + }, + channelPartial: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + {StreamID: streamIDPartial, Aggregator: llotypes.AggregatorMedian}, + }, + }, + } + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, + WithOracles(oracles), WithOffchainConfig(offchainConfig), WithV31(), + ) + + // collectChannels drains transmitted JSON reports for the given duration and + // returns the set of channel IDs seen. + collectChannels := func(d time.Duration) map[uint32]bool { + seen := make(map[uint32]bool) + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + pckt, errReceive := receiveWithTimeout(t, packetCh, time.Until(deadline)) + if errReceive != nil { + break + } + req := pckt.req + if req.ReportFormat != uint32(llotypes.ReportFormatJSON) { + continue + } + _, _, r, _, errDecode := (lloreportcodec.JSONReportCodec{}).UnpackDecode(req.Payload) + if errDecode != nil { + continue + } + seen[r.ChannelID] = true + } + return seen + } + + // The fully observed channel must report regardless of the floor. This also + // establishes that the DON has reached a reporting steady state before + // judging the under-observed channel. + require.Eventually(t, func() bool { + return collectChannels(2 * time.Second)[channelFull] + }, reportTimeout, 100*time.Millisecond, "fully observed channel should produce reports") + + // Now judge the under-observed channel over a window that spans many rounds. + seen := collectChannels(15 * time.Second) + require.True(t, seen[channelFull], "fully observed channel should keep reporting") + if expectPartialReports { + require.True(t, seen[channelPartial], + "under-observed channel should report when the contribution floor is met") + } else { + require.False(t, seen[channelPartial], + "under-observed channel should not report when it cannot meet the contribution floor") + } +} + +// TestIntegration_LLO_v31_blob_catchup covers the v31 blob path for an oracle +// that is not present for the earlier rounds. v31 disseminates stream values +// through blobs, so such an oracle can only contribute once it has fetched the +// blobs and state the running DON produced without it. +// +// Two ways in: an oracle whose LLO job is created after the DON is already +// reporting, and an oracle whose job is deleted mid-run and re-created after +// the DON has advanced many rounds without it. In both cases the remaining +// 3 oracles (2F+1) keep the DON reporting throughout. +func TestIntegration_LLO_v31_blob_catchup(t *testing.T) { + t.Parallel() + tcs := []struct { + name string + // rejoin starts the lagging oracle with an LLO job and deletes it + // mid-run, instead of never creating it until catch-up time. + rejoin bool + salt int64 + donID uint32 + }{ + {name: "oracle joining after the DON is reporting catches up", rejoin: false, salt: 800, donID: uint32(555333)}, + {name: "oracle rejoining after its job is deleted catches up", rejoin: true, salt: 850, donID: uint32(555444)}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testIntegrationLLOV31BlobCatchup(t, tc.rejoin, tc.salt, tc.donID) + }) + } +} + +func testIntegrationLLOV31BlobCatchup(t *testing.T, rejoin bool, salt int64, donID uint32) { + const ( + streamID = uint32(790) + channelID = uint32(1) + ) + // laggingNode is the index of the oracle that misses rounds. + laggingNode := nNodes - 1 + + offchainConfig := lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, + EnableObservationCompression: true, + } + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + for i := range nNodes { + k := big.NewInt(salt + int64(i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress, _, _, _, _ := setupBlockchain(t) + fromBlock := 1 + + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo_catchup", backend, bootstrapCSAKey, nil) + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + packetCh := make(chan *packet, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, serverKey, packetCh) + serverURL := startMercuryServer(t, srv, clientPubKeys) + + oracles, nodes := setupNodes(t, nNodes, backend, clientCSAKeys, func(c *chainlink.Config) { + c.Mercury.Transmitter.Protocol = new(mercurytransmitter.MercuryTransmitterProtocolGRPC) + }) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, fmt.Sprintf("job-catchup-%d", donID), relayType, relayConfig) + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d +pluginVersion = "v31"`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + + // One counter per node so the lagging oracle's observations can be told + // apart from the rest of the DON's. + bridgeCalls := make([]*atomic.Uint64, nNodes) + price := decimal.NewFromFloat(123.4) + addLaggingLLOJob := func() int32 { + return addLLOJob( + t, + nodes[laggingNode], + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[laggingNode], + "blob-catchup-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + var laggingJobID int32 + for i, node := range nodes { + bridgeCalls[i] = new(atomic.Uint64) + bridge := createSingleDecimalCountingBridge(t, "catchup", i, price, node.App.BridgeORM(), bridgeCalls[i]) + addSingleDecimalStreamJob(t, node, streamID, bridge) + if i == laggingNode { + // The late joiner gets no LLO job yet; the rejoining oracle gets one + // that is deleted below. + if rejoin { + laggingJobID = addLaggingLLOJob() + } + continue + } + addLLOJob( + t, + node, + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[i], + "blob-catchup-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + channelDefinitions := llotypes.ChannelDefinitions{ + channelID: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + {StreamID: streamID, Aggregator: llotypes.AggregatorMedian}, + }, + }, + } + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, + WithOracles(oracles), WithOffchainConfig(offchainConfig), WithV31(), + ) + + // reportsFlowing drains transmitted reports for the channel under test until + // it sees one or the deadline passes. + reportsFlowing := func(d time.Duration) bool { + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + pckt, errReceive := receiveWithTimeout(t, packetCh, time.Until(deadline)) + if errReceive != nil { + return false + } + req := pckt.req + if req.ReportFormat != uint32(llotypes.ReportFormatJSON) { + continue + } + _, _, r, _, errDecode := (lloreportcodec.JSONReportCodec{}).UnpackDecode(req.Payload) + if errDecode == nil && r.ChannelID == channelID { + return true + } + } + return false + } + + require.Eventually(t, func() bool { + return reportsFlowing(2 * time.Second) + }, reportTimeout, 100*time.Millisecond, "DON should report before the lagging oracle joins") + + if rejoin { + nodes[laggingNode].DeleteJob(t, laggingJobID) + // Wait for the deleted oracle to stop observing, so the rounds that + // follow genuinely advance without it. + var last uint64 + var stableSince time.Time + require.Eventually(t, func() bool { + c := bridgeCalls[laggingNode].Load() + if c != last { + last = c + stableSince = time.Now() + return false + } + return time.Since(stableSince) > 2*time.Second + }, 30*time.Second, 100*time.Millisecond, "deleted oracle should stop observing") + // Let the remaining oracles advance a good number of rounds alone. + require.True(t, reportsFlowing(10*time.Second), "DON should keep reporting without the deleted oracle") + } + + callsBeforeCatchup := bridgeCalls[laggingNode].Load() + if !rejoin { + require.Zero(t, callsBeforeCatchup, "oracle without an LLO job should not observe") + } + + laggingJobID = addLaggingLLOJob() + require.NotZero(t, laggingJobID) + + require.Eventually(t, func() bool { + return bridgeCalls[laggingNode].Load() > callsBeforeCatchup + }, reportTimeout, 100*time.Millisecond, "the lagging oracle should observe once it has caught up") + + require.True(t, reportsFlowing(10*time.Second), "DON should keep reporting after the lagging oracle catches up") +} + +// TestIntegration_LLO_v31_restart_resumes restarts one oracle of a reporting +// v31 DON. v31 keeps its protocol state in a pebble key-value store on disk +// rather than in an outcome carried between rounds, so a restarted oracle has +// to reopen that store, recover its LLO job from the database, and rejoin a +// protocol that advanced without it. +// +// The remaining 3 oracles (2F+1) keep the DON reporting throughout, so a +// restart that failed to recover would show up as an oracle that never resumes +// observing rather than as a stalled DON. +func TestIntegration_LLO_v31_restart_resumes(t *testing.T) { + t.Parallel() + + const ( + salt = int64(900) + donID = uint32(555555) + streamID = uint32(890) + channelID = uint32(1) + ) + // restartedNode is the index of the oracle that is stopped and started again. + restartedNode := nNodes - 1 + + offchainConfig := lloprotocol.OffchainConfig{ + ProtocolVersion: 1, + DefaultMinReportIntervalNanoseconds: uint64(1 * time.Second), + AggregationFaultTolerance: aggregationFaultTolerance, + EnableObservationCompression: true, + } + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + for i := range nNodes { + k := big.NewInt(salt + int64(i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress, _, _, _, _ := setupBlockchain(t) + fromBlock := 1 + + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo_restart", backend, bootstrapCSAKey, nil) + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + packetCh := make(chan *packet, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, serverKey, packetCh) + serverURL := startMercuryServer(t, srv, clientPubKeys) + + oracles, nodes, restarts := setupNodesWithRestart(t, nNodes, backend, clientCSAKeys, func(c *chainlink.Config) { + c.Mercury.Transmitter.Protocol = new(mercurytransmitter.MercuryTransmitterProtocolGRPC) + }) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, "job-restart", relayType, relayConfig) + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d +pluginVersion = "v31"`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + + // One counter per node, so the restarted oracle's observations can be told + // apart from the rest of the DON's. + bridgeCalls := make([]*atomic.Uint64, nNodes) + price := decimal.NewFromFloat(456.7) + for i, node := range nodes { + bridgeCalls[i] = new(atomic.Uint64) + bridge := createSingleDecimalCountingBridge(t, "restart", i, price, node.App.BridgeORM(), bridgeCalls[i]) + addSingleDecimalStreamJob(t, node, streamID, bridge) + addLLOJob( + t, + node, + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[i], + "restart-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + channelDefinitions := llotypes.ChannelDefinitions{ + channelID: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + {StreamID: streamID, Aggregator: llotypes.AggregatorMedian}, + }, + }, + } + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, + WithOracles(oracles), WithOffchainConfig(offchainConfig), WithV31(), + ) + + reportsFlowing := func(d time.Duration) bool { + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + pckt, errReceive := receiveWithTimeout(t, packetCh, time.Until(deadline)) + if errReceive != nil { + return false + } + req := pckt.req + if req.ReportFormat != uint32(llotypes.ReportFormatJSON) { + continue + } + _, _, r, _, errDecode := (lloreportcodec.JSONReportCodec{}).UnpackDecode(req.Payload) + if errDecode == nil && r.ChannelID == channelID { + return true + } + } + return false + } + + require.Eventually(t, func() bool { + return reportsFlowing(2 * time.Second) + }, reportTimeout, 100*time.Millisecond, "DON should report before the restart") + require.Positive(t, bridgeCalls[restartedNode].Load(), "the oracle to restart should be observing first") + + // Restart. The new application reopens the same database and key-value + // store root, so nothing below re-creates the jobs. + var callsWhileDown uint64 + app := restarts[restartedNode](t, func() { + // While the oracle is down it must not observe, which is what makes the + // resumption below evidence of a real restart. + settled := bridgeCalls[restartedNode].Load() + require.Eventually(t, func() bool { + c := bridgeCalls[restartedNode].Load() + if c != settled { + settled = c + return false + } + return true + }, 15*time.Second, 500*time.Millisecond, "a stopped oracle should stop observing") + callsWhileDown = settled + require.True(t, reportsFlowing(10*time.Second), "DON should keep reporting while the oracle is down") + require.Equal(t, callsWhileDown, bridgeCalls[restartedNode].Load(), "a stopped oracle should not observe") + }) + nodes[restartedNode].App = app + + callsAfterRestart := bridgeCalls[restartedNode].Load() + require.Equal(t, callsWhileDown, callsAfterRestart, "no observations should have happened while the oracle was down") + require.Eventually(t, func() bool { + return bridgeCalls[restartedNode].Load() > callsAfterRestart + }, reportTimeout, 100*time.Millisecond, "the restarted oracle should resume observing without its jobs being re-created") + + require.True(t, reportsFlowing(10*time.Second), "DON should keep reporting after the restarted oracle rejoins") +} diff --git a/core/services/workflows/syncer/v2/engine_registry.go b/core/services/workflows/syncer/v2/engine_registry.go index 82eba6d951d..f360a1f0132 100644 --- a/core/services/workflows/syncer/v2/engine_registry.go +++ b/core/services/workflows/syncer/v2/engine_registry.go @@ -9,6 +9,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/hashutil" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" ) var ErrNotFound = errors.New("engine not found") @@ -23,6 +24,13 @@ type ServiceWithMetadata struct { services.Service } +// Coordinated is true for an engine that leaves trigger registration, +// dequeuing and acknowledgement to the TriggerCoordinator. +func (e ServiceWithMetadata) Coordinated() bool { + engine, ok := e.Service.(v2.WorkflowEngine) + return ok && engine.IsCoordinated() +} + // engineEntry holds the engine and its associated source for internal storage. type engineEntry struct { engine services.Service diff --git a/core/services/workflows/syncer/v2/engine_registry_test.go b/core/services/workflows/syncer/v2/engine_registry_test.go index 09ca7e02850..92270c0edb6 100644 --- a/core/services/workflows/syncer/v2/engine_registry_test.go +++ b/core/services/workflows/syncer/v2/engine_registry_test.go @@ -3,11 +3,13 @@ package v2 import ( "context" "testing" + "time" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" ) func TestEngineRegistry(t *testing.T) { @@ -168,3 +170,109 @@ func (f fakeService) Ready() error { return nil } func (f fakeService) HealthReport() map[string]error { return map[string]error{} } func (f fakeService) Name() string { return "" } + +// fakeDrainableService reports a fixed drain state; it is otherwise a fakeService. +type fakeDrainableService struct { + fakeService + draining bool +} + +func (f *fakeDrainableService) Drain() bool { return false } + +func (f *fakeDrainableService) ActiveExecutions() int32 { return 0 } + +func (f *fakeDrainableService) DrainStartedAt() (time.Time, bool) { + if !f.draining { + return time.Time{}, false + } + return time.Unix(1, 0), true +} + +// fakeCoordinatedDrainableEngine is a v2.WorkflowEngine that reports as +// coordinated and drainable. Only the methods ServiceWithMetadata.Coordinated +// and the DrainableService assertion touch are implemented. +type fakeCoordinatedDrainableEngine struct { + v2.WorkflowEngine + draining bool +} + +func (f *fakeCoordinatedDrainableEngine) IsCoordinated() bool { return true } + +func (f *fakeCoordinatedDrainableEngine) Drain() bool { return false } + +func (f *fakeCoordinatedDrainableEngine) ActiveExecutions() int32 { return 0 } + +func (f *fakeCoordinatedDrainableEngine) DrainStartedAt() (time.Time, bool) { + if !f.draining { + return time.Time{}, false + } + return time.Unix(1, 0), true +} + +func TestEngineRegistry_Coordinated(t *testing.T) { + t.Parallel() + legacyID := types.WorkflowID([32]byte{1}) + coordinatedID := types.WorkflowID([32]byte{2}) + + er := NewEngineRegistry() + require.NoError(t, er.AddWithReconcileKey(legacyID, "TestSource", "legacy-key", &fakeService{})) + require.NoError(t, er.AddWithReconcileKey(coordinatedID, "TestSource", "coordinated-key", &fakeCoordinatedDrainableEngine{})) + require.ErrorIs(t, er.AddWithReconcileKey(coordinatedID, "TestSource", "coordinated-key", &fakeCoordinatedDrainableEngine{}), ErrAlreadyExists) + + entry, ok := er.Get(coordinatedID) + require.True(t, ok) + require.True(t, entry.Coordinated()) + require.Equal(t, "coordinated-key", entry.ReconcileKey) + + entry, ok = er.Get(legacyID) + require.True(t, ok) + require.False(t, entry.Coordinated()) + + byID := map[types.WorkflowID]bool{} + for _, e := range er.GetAll() { + byID[e.WorkflowID] = e.Coordinated() + } + require.Equal(t, map[types.WorkflowID]bool{legacyID: false, coordinatedID: true}, byID) + + for _, e := range er.GetBySource("TestSource") { + require.Equal(t, e.WorkflowID == coordinatedID, e.Coordinated()) + } + + popped, err := er.Pop(coordinatedID) + require.NoError(t, err) + require.True(t, popped.Coordinated()) + + for _, e := range er.PopAll() { + require.False(t, e.Coordinated()) + } +} + +func TestCountEngines(t *testing.T) { + t.Parallel() + + t.Run("empty registry", func(t *testing.T) { + t.Parallel() + require.Equal(t, engineCounts{}, countEngines(nil)) + }) + + t.Run("mixed engine types", func(t *testing.T) { + t.Parallel() + engines := []ServiceWithMetadata{ + {Service: &fakeCoordinatedDrainableEngine{}}, + {Service: &fakeCoordinatedDrainableEngine{draining: true}}, + {Service: &fakeDrainableService{}}, + {Service: &fakeDrainableService{draining: true}}, + {Service: &fakeService{}}, + {Service: newTestShardFailoverManager(t, &mockEngine{})}, + } + require.Equal(t, engineCounts{draining: 2, coordinated: 2, legacy: 4}, countEngines(engines)) + }) + + t.Run("a sharded wrapper follows the engine it wraps", func(t *testing.T) { + t.Parallel() + engines := []ServiceWithMetadata{ + {Service: newTestShardFailoverManager(t, &fakeCoordinatedDrainableEngine{})}, + } + require.Equal(t, engineCounts{coordinated: 1}, countEngines(engines)) + }) +} diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index f0ffcae60d5..cbb69bd4e6c 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -56,11 +56,9 @@ type ORM interface { artifacts.WorkflowSpecsDS } -// engineFactoryFn creates a workflow engine. The initDone channel is used to signal when the engine -// has completed initialization (including trigger subscriptions). For v2 engines, this is wired to -// the OnInitialized lifecycle hook. For v1 legacy DAG engines, nil is sent immediately after engine -// creation since they don't support async initialization hooks. -type engineFactoryFn func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) +// engineFactoryFn creates a workflow engine. The initDone channel is used to signal when +// the engine has completed initialization; it is wired to the OnInitialized lifecycle hook. +type engineFactoryFn func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) type DrainableService interface { Drain() bool @@ -77,19 +75,26 @@ type eventHandler struct { lggr logger.Logger - workflowStore store.Store - capRegistry registry.CapabilitiesRegistry - executionHandlers *confidentialrelay.ExecutionHandlers - donTimeStore *dontime.Store - useLocalTimeProvider bool - engineRegistry *EngineRegistry - emitter custmsg.MessageEmitter - emitterMu sync.RWMutex - engineFactory engineFactoryFn - engineLimiters *v2.EngineLimiters - featureFlags *v2.EngineFeatureFlags - ratelimiter *ratelimiter.RateLimiter - workflowLimits limits.ResourceLimiter[int] + workflowStore store.Store + capRegistry registry.CapabilitiesRegistry + executionHandlers *confidentialrelay.ExecutionHandlers + donTimeStore *dontime.Store + useLocalTimeProvider bool + engineRegistry *EngineRegistry + emitter custmsg.MessageEmitter + emitterMu sync.RWMutex + engineFactory engineFactoryFn + engineLimiters *v2.EngineLimiters + featureFlags *v2.EngineFeatureFlags + ratelimiter *ratelimiter.RateLimiter + workflowLimits limits.ResourceLimiter[int] + + // triggerCoordinator owns registration/handles/ACK for every workflow + // running the coordinated engine. Nil until wired via WithTriggerCoordinator. + // useCoordinatedEngine picks the legacy path when nil, regardless of the + // CoordinatedEngine flag. + triggerCoordinator TriggerCoordinator + workflowArtifactsStore WorkflowArtifactsStore workflowEncryptionKey workflowkey.Key workflowDonSubscriber capabilities.DonSubscriber @@ -152,9 +157,9 @@ func WithEngineFactoryFn(efn engineFactoryFn) func(*eventHandler) { } } -func WithStaticEngine(engine services.Service) func(*eventHandler) { +func WithStaticEngine(engine v2.WorkflowEngine) func(*eventHandler) { return func(e *eventHandler) { - e.engineFactory = func(_ context.Context, _, _ string, _ types.WorkflowName, _ string, _, _ []byte, _ string, initDone chan<- error) (services.Service, error) { + e.engineFactory = func(_ context.Context, _, _ string, _ types.WorkflowName, _ string, _, _ []byte, _ string, initDone chan<- error) (v2.WorkflowEngine, error) { // For static engines (used in tests), signal immediate initialization success if initDone != nil { initDone <- nil @@ -164,6 +169,16 @@ func WithStaticEngine(engine services.Service) func(*eventHandler) { } } +// WithTriggerCoordinator wires the TriggerCoordinator used for every workflow +// the CoordinatedEngine flag routes to the coordinated engine. Without +// this option, tryEngineCreate always takes the legacy path, regardless of +// the flag's value. +func WithTriggerCoordinator(tc TriggerCoordinator) func(*eventHandler) { + return func(e *eventHandler) { + e.triggerCoordinator = tc + } +} + func WithBillingClient(client metering.BillingClient) func(*eventHandler) { return func(e *eventHandler) { e.billingClient = client @@ -367,15 +382,15 @@ func NewEventHandler( return nil, fmt.Errorf("new metrics: %w", metricsErr) } eh.metrics = metricsInst - eh.engineFactory = eh.engineFactoryFn + eh.engineFactory = eh.newEngine for _, o := range opts { o(eh) } eh.Service, eh.eng = services.Config{ Name: "EventHandler", - // The workflow store and the spec meter are started and stopped - // alongside the handler. + // The workflow store, the spec meter and the trigger coordinator are + // started and stopped alongside the handler. NewSubServices: func(logger.Logger) []services.Service { var subs []services.Service if eh.workflowStore != nil { @@ -384,6 +399,9 @@ func NewEventHandler( if eh.specMeter != nil { subs = append(subs, eh.specMeter) } + if eh.triggerCoordinator != nil { + subs = append(subs, eh.triggerCoordinator) + } return subs }, Start: eh.start, @@ -855,9 +873,32 @@ func (h *eventHandler) fetchOrganizationID(ctx context.Context, workflowOwner st return organizationID, nil } -func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) { +// useCoordinatedEngine reads the CoordinatedEngine flag. A node without a +// TriggerCoordinator always takes the legacy path, and so does a sharded node: +// sharding and the coordinated engine are not yet compatible. +func (h *eventHandler) useCoordinatedEngine(ctx context.Context, workflowID string) bool { + if h.triggerCoordinator == nil || h.featureFlags == nil || h.featureFlags.CoordinatedEngine == nil { + return false + } + if h.shardingEnabled && h.dispatcher != nil { + return false + } + open, err := h.featureFlags.CoordinatedEngine.IsOpen(ctx) + if err != nil { + h.lggr.Warnw("Could not evaluate CoordinatedEngine flag, falling back to the legacy engine", + "workflowID", workflowID, "err", err) + return false + } + return open +} + +// buildEngineConfig builds the module stack (local WASM module plus the +// confidential module) and the EngineConfig that the legacy and the +// coordinated engine share. +func (h *eventHandler) buildEngineConfig(ctx context.Context, workflowID, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (*v2.EngineConfig, error) { lggr := logger.Named(h.lggr, "WorkflowEngine.Module") lggr = logger.With(lggr, "workflowID", workflowID, "workflowName", name, "workflowOwner", owner) + var sdkName string moduleConfig := &host.ModuleConfig{ Logger: lggr, @@ -888,7 +929,6 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID, owner st h.lggr.Debugw("Finished creating module for workflowID", "workflowID", workflowID) - // V2 aka "NoDAG" // Wrap the local WASM module in a RequirementSelectingModule that routes // triggers with a TEE requirement to the ConfidentialModule (which delegates // to the confidential-workflows capability and runs the WASM inside the @@ -912,6 +952,21 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID, owner st h.wireInitDoneHook(cfg, initDone) + return cfg, nil +} + +// newEngine is the default engineFactoryFn: it builds the module stack and +// EngineConfig, wires sharding, and calls one of the two engine constructors. +// +// The coordinated engine registers no triggers itself: +// tryCoordinatedEngineCreate hands it to the coordinator once it is in the +// registry, and the coordinator calls Subscribe to obtain its subscriptions. +func (h *eventHandler) newEngine(ctx context.Context, workflowID, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) { + cfg, err := h.buildEngineConfig(ctx, workflowID, owner, name, tag, config, binary, binaryURL, initDone) + if err != nil { + return nil, err + } + var manager *ShardFailoverManager if h.shardingEnabled && h.dispatcher != nil { if h.shardFailoverComm == nil { @@ -935,7 +990,18 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID, owner st manager.WireHooks(cfg) } - engine, err := v2.NewEngine(cfg) + construct := v2.NewEngine + if h.useCoordinatedEngine(ctx, workflowID) { + // The trigger coordinator is not implemented yet, so a workflow routed here + // will never receive triggers. Warn once per engine creation with the workflowID + h.lggr.Warnw("Routing workflow to the coordinated engine; trigger delivery is not implemented yet", + "workflowID", workflowID, "workflowName", name, "workflowOwner", owner) + + cfg.TriggerAcknowledger = h.triggerCoordinator + construct = v2.NewCoordinatedEngine + } + + engine, err := construct(cfg) if err != nil { return nil, err } @@ -981,6 +1047,14 @@ func (h *eventHandler) stopEngine(ctx context.Context, workflowID types.Workflow e, ok := h.engineRegistry.Get(workflowID) var drainable DrainableService if ok { + // on coordinated engines, stop coordinator ingress before draining, + // so the drain can actually reach zero active executions. + if e.Coordinated() && h.triggerCoordinator != nil { + if err := h.triggerCoordinator.UnregisterTriggers(workflowID.Hex()); err != nil && !errors.Is(err, ErrWorkflowNotCoordinated) { + h.lggr.Errorw("Failed to unregister triggers via coordinator", "workflowID", workflowID.String(), "err", err) + } + } + var isDrainable bool if drainable, isDrainable = e.Service.(DrainableService); isDrainable { if started := drainable.Drain(); started { @@ -1106,6 +1180,16 @@ func (h *eventHandler) tryEngineCleanup(workflowID types.WorkflowID) error { return nil } + // Unregister before close, same as stopEngine. + // This path (reconcile-to-inactive, replace-draining-engine) must mirror + // stopEngine's coordinator handling, or trigger registrations are left + // orphaned on this path alone. + if e.Coordinated() && h.triggerCoordinator != nil { + if err := h.triggerCoordinator.UnregisterTriggers(workflowID.Hex()); err != nil && !errors.Is(err, ErrWorkflowNotCoordinated) { + h.lggr.Errorw("Failed to unregister triggers via coordinator", "workflowID", workflowID.String(), "err", err) + } + } + // Close the engine, then remove it from the registry only once cleanup has succeeded. The // registry entry is what tells us this workflow still needs cleanup, so if a step fails we // leave it in place and return the error, allowing a future attempt to retry in case the @@ -1134,9 +1218,22 @@ func (h *eventHandler) cleanupModuleCache(workflowID string) { } } +// engineInputs is the validated, decoded form of a WorkflowSpec that both +// engine creation paths start from. +type engineInputs struct { + wid types.WorkflowID + workflowName types.WorkflowName + binary []byte + config []byte + reconcileKey string +} + // tryEngineCreate attempts to create a new workflow engine, start it, and register it with the engine registry. // This function waits for the engine to complete initialization (including trigger subscriptions) before returning, // ensuring that the workflowActivated event accurately reflects the deployment status including trigger registration. +// +// The CoordinatedEngine flag is read here, once, and decides between the legacy +// and the coordinated creation path. func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSpec, source string) error { ctx, span := h.tracer.Start(ctx, "engine_create", trace.WithAttributes( @@ -1146,15 +1243,102 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp )) defer span.End() + in, err := h.prepareEngineInputs(ctx, spec) + if err != nil { + return err + } + + initDone := make(chan error, 1) + engine, err := h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, in.workflowName, spec.WorkflowTag, in.config, in.binary, spec.BinaryURL, initDone) + if err != nil { + return fmt.Errorf("failed to create workflow engine: %w", err) + } + + if engine.IsCoordinated() { + return h.tryCoordinatedEngineCreate(ctx, spec, source, in, engine, initDone) + } + return h.tryLegacyEngineCreate(ctx, spec, source, in, engine, initDone) +} + +// tryLegacyEngineCreate finishes creation for an engine that registers its own +// triggers while it starts, so it is done once the engine is in the registry. +func (h *eventHandler) tryLegacyEngineCreate(ctx context.Context, spec *job.WorkflowSpec, source string, in *engineInputs, engine v2.WorkflowEngine, initDone <-chan error) error { + if err := h.startEngine(ctx, spec.WorkflowID, engine, initDone); err != nil { + return err + } + + if err := h.engineRegistry.AddWithReconcileKey(in.wid, source, in.reconcileKey, engine); err != nil { + return h.registryAddFailed(in.wid, source, engine, err) + } + return nil +} + +// tryCoordinatedEngineCreate finishes creation for an engine that registers no triggers itself. +func (h *eventHandler) tryCoordinatedEngineCreate(ctx context.Context, spec *job.WorkflowSpec, source string, in *engineInputs, engine v2.WorkflowEngine, initDone <-chan error) error { + if err := h.startEngine(ctx, spec.WorkflowID, engine, initDone); err != nil { + return err + } + + if err := h.engineRegistry.AddWithReconcileKey(in.wid, source, in.reconcileKey, engine); err != nil { + return h.registryAddFailed(in.wid, source, engine, err) + } + + cleanup := func(location string) { + if _, innerErr := h.engineRegistry.Pop(in.wid); innerErr != nil { + h.lggr.Warnw(fmt.Sprintf("Failed to pop engine from registry after %s failure", location), "workflowID", in.wid.Hex(), "err", innerErr) + } + if innerErr := engine.Close(); innerErr != nil { + h.lggr.Warnw(fmt.Sprintf("Failed to close engine after %s failure", location), "workflowID", in.wid.Hex(), "err", innerErr) + } + } + + // Registration happens here, now that the engine is already in the + // registry, so the coordinator's readers can resolve it for the first + // event. RegisterTriggers calls the engine's Subscribe itself — the WASM + // call is scoped to the one workflow it describes, which matters because + // Handle runs on a worker pool and several workflows reach this point + // concurrently. + localNode, err := h.capRegistry.LocalNode(ctx) + if err != nil { + cleanup("local node resolution") + return fmt.Errorf("failed to resolve local node DON ID for trigger registration: %w", err) + } + donID := localNode.WorkflowDON.ID + + triggerIDs, err := h.triggerCoordinator.RegisterTriggers(ctx, engine, RegistrationParams{ + WorkflowOwner: spec.WorkflowOwner, + WorkflowName: in.workflowName.Hex(), + DecodedWorkflowName: in.workflowName.String(), + WorkflowTag: spec.WorkflowTag, + WorkflowDonID: donID, + // pinnedWorkflowDonConfigVersion in v2 pins this to 1 to avoid forcing + // forwarder updates on config churn; mirrored here since the syncer + // can't reference that unexported v2 constant. + WorkflowDonConfigVersion: 1, + WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, + WorkflowRegistryAddress: h.workflowRegistryAddress, + }) + if err != nil { + cleanup("trigger registration") + return fmt.Errorf("failed to register triggers via coordinator: %w", err) + } + h.lggr.Infow("Registered triggers via coordinator", "workflowID", in.wid.Hex(), "triggerIDs", triggerIDs) + return nil +} + +// prepareEngineInputs decodes and validates the spec before any engine is +// built: the binary, owner and name must decode, and the workflow ID must +// match the one derived from the stored artifacts. +func (h *eventHandler) prepareEngineInputs(ctx context.Context, spec *job.WorkflowSpec) (*engineInputs, error) { // Ensure the capabilities registry is ready before creating any Engine instances. // This should be guaranteed by the Workflow Registry Syncer. if err := h.ensureCapRegistryReady(ctx); err != nil { - return fmt.Errorf("failed to ensure capabilities registry is ready: %w", err) + return nil, fmt.Errorf("failed to ensure capabilities registry is ready: %w", err) } decodedBinary, err := hex.DecodeString(spec.Workflow) if err != nil { - return nonRetryable(fmt.Errorf("failed to decode workflow spec binary: %w", err)) + return nil, nonRetryable(fmt.Errorf("failed to decode workflow spec binary: %w", err)) } // Free the hex-encoded binary string as it is not needed beyond this decode spec.Workflow = "" @@ -1166,88 +1350,91 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp // Workflow ID should match what is generated from the stored artifacts ownerBytes, err := hex.DecodeString(spec.WorkflowOwner) if err != nil { - return nonRetryable(fmt.Errorf("failed to decode owner: %w", err)) + return nil, nonRetryable(fmt.Errorf("failed to decode owner: %w", err)) } configBytes := []byte(spec.Config) hash, err := pkgworkflows.GenerateWorkflowID(ownerBytes, spec.WorkflowName, decodedBinary, configBytes, secretsURL) if err != nil { - return fmt.Errorf("failed to generate workflow id: %w", err) + return nil, fmt.Errorf("failed to generate workflow id: %w", err) } wid, err := types.WorkflowIDFromHex(spec.WorkflowID) if err != nil { - return nonRetryable(fmt.Errorf("invalid workflow id: %w", err)) + return nil, nonRetryable(fmt.Errorf("invalid workflow id: %w", err)) } if !types.WorkflowID(hash).Equal(wid) { - return nonRetryable(fmt.Errorf("workflowID mismatch: %s != %s", types.WorkflowID(hash).Hex(), wid.Hex())) + return nil, nonRetryable(fmt.Errorf("workflowID mismatch: %s != %s", types.WorkflowID(hash).Hex(), wid.Hex())) } - // Start a new WorkflowEngine instance, and add it to local engine registry workflowName, err := types.NewWorkflowName(spec.WorkflowName) if err != nil { - return nonRetryable(fmt.Errorf("invalid workflow name: %w", err)) + return nil, nonRetryable(fmt.Errorf("invalid workflow name: %w", err)) } - // Create a channel to receive the initialization result. - // This allows us to wait for the engine to complete initialization (including trigger subscriptions) - // before emitting the workflowActivated event, ensuring the event accurately reflects deployment status. - initDone := make(chan error, 1) - var engine services.Service - - engine, err = h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, configBytes, decodedBinary, spec.BinaryURL, initDone) + // Identity fingerprint stored alongside the engine in the registry. + reconcileKey, err := ReconcileKey(ownerBytes, spec.WorkflowName) if err != nil { - return fmt.Errorf("failed to create workflow engine: %w", err) + return nil, fmt.Errorf("failed to compute reconcile key: %w", err) } - if err = engine.Start(ctx); err != nil { + return &engineInputs{ + wid: wid, + workflowName: workflowName, + binary: decodedBinary, + config: configBytes, + reconcileKey: reconcileKey, + }, nil +} + +// startEngine starts engine and waits for it to complete initialization +// (including trigger subscriptions), so we don't emit workflowActivated events +// before the engine initializes successfully. The engine is closed if +// initialization fails or ctx is cancelled first. +func (h *eventHandler) startEngine(ctx context.Context, workflowID string, engine services.Service, initDone <-chan error) error { + if err := engine.Start(ctx); err != nil { return fmt.Errorf("failed to start workflow engine: %w", err) } - // Wait for the engine to complete initialization (including trigger subscriptions). - // This ensures we don't emit workflowActivated events before the engine initializes successfully. select { case <-ctx.Done(): // Context cancelled while waiting for initialization if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", spec.WorkflowID) + h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("context cancelled while waiting for engine initialization: %w", ctx.Err()) case initErr := <-initDone: if initErr != nil { // Engine initialization failed (e.g., trigger subscription failed) if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", spec.WorkflowID) + h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("engine initialization failed: %w", initErr) } } + return nil +} - // Engine is fully initialized, add to registry with source tracking and identity fingerprint - reconcileKey, err := ReconcileKey(ownerBytes, spec.WorkflowName) - if err != nil { - return fmt.Errorf("failed to compute reconcile key: %w", err) - } - if err := h.engineRegistry.AddWithReconcileKey(wid, source, reconcileKey, engine); err != nil { - if closeErr := engine.Close(); closeErr != nil { - return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) - } - - // Check for WorkflowID collision across sources - if errors.Is(err, ErrAlreadyExists) { - existingEntry, found := h.engineRegistry.Get(wid) - if found { - h.lggr.Warnw("WorkflowID collision detected: workflow already exists from different source", - "workflowID", wid.Hex(), - "attemptedSource", source, - "existingSource", existingEntry.Source, - "hint", "Each workflow ID should only be registered from a single source. Check your workflow configurations for duplicates.") - } +// registryAddFailed closes an engine the registry refused and returns the +// error to report. Both creation paths share it. +func (h *eventHandler) registryAddFailed(wid types.WorkflowID, source string, engine services.Service, err error) error { + if closeErr := engine.Close(); closeErr != nil { + return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) + } + + // Check for WorkflowID collision across sources + if errors.Is(err, ErrAlreadyExists) { + existingEntry, found := h.engineRegistry.Get(wid) + if found { + h.lggr.Warnw("WorkflowID collision detected: workflow already exists from different source", + "workflowID", wid.Hex(), + "attemptedSource", source, + "existingSource", existingEntry.Source, + "hint", "Each workflow ID should only be registered from a single source. Check your workflow configurations for duplicates.") } - - // This shouldn't happen because we call the handler serially and - // check for running engines above, see the call to engineRegistry.Contains. - return fmt.Errorf("invariant violation: %w", err) } - return nil + + // This shouldn't happen because we call the handler serially and + // check for running engines above, see the call to engineRegistry.Contains. + return fmt.Errorf("invariant violation: %w", err) } func (h *eventHandler) overrideFetcherForOwner(owner string) v2.SecretsFetcher { diff --git a/core/services/workflows/syncer/v2/handler_engine_create_test.go b/core/services/workflows/syncer/v2/handler_engine_create_test.go new file mode 100644 index 00000000000..50939ee7921 --- /dev/null +++ b/core/services/workflows/syncer/v2/handler_engine_create_test.go @@ -0,0 +1,276 @@ +package v2 + +import ( + "context" + "encoding/hex" + "errors" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace/noop" + + capreg "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + pkgworkflows "github.com/smartcontractkit/chainlink-common/pkg/workflows" + "github.com/smartcontractkit/chainlink/v2/core/services/job" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" +) + +// fakeCoordinatedEngine is a v2.WorkflowEngine that reports as coordinated. +// Only the lifecycle methods tryEngineCreate calls are implemented. +type fakeCoordinatedEngine struct { + v2.WorkflowEngine + closed atomic.Bool +} + +func (f *fakeCoordinatedEngine) Start(context.Context) error { return nil } + +func (f *fakeCoordinatedEngine) Close() error { + f.closed.Store(true) + return nil +} + +func (f *fakeCoordinatedEngine) IsCoordinated() bool { return true } + +// recordingCoordinator records the subscribers handed to RegisterTriggers. +type recordingCoordinator struct { + TriggerCoordinator + registerErr error + registered []v2.Subscriber +} + +func (c *recordingCoordinator) RegisterTriggers(_ context.Context, subscriber v2.Subscriber, _ RegistrationParams) ([]string, error) { + c.registered = append(c.registered, subscriber) + return nil, c.registerErr +} + +// nodeConfig is the node state that decides the routing policy. +type nodeConfig struct { + flagOpen bool + withCoordinator bool + sharded bool +} + +// engineCreateFixture is an eventHandler whose engine factory mirrors the real +// one's contract: it reads the routing policy itself and returns the matching +// engine, leaving tryEngineCreate to route on the engine's IsCoordinated. +type engineCreateFixture struct { + h *eventHandler + legacyEngine v2.WorkflowEngine + coordinatedEngine *fakeCoordinatedEngine + factoryCalls atomic.Int32 + factoryErr error + coordinator *recordingCoordinator +} + +func newEngineCreateFixture(t *testing.T, cfg nodeConfig) *engineCreateFixture { + t.Helper() + lggr := logger.Test(t) + registry := capreg.NewRegistry(lggr) + registry.SetRegistryMetadata(&capreg.TestRegistryMetadata{}) + + f := &engineCreateFixture{ + legacyEngine: &mockEngine{}, + coordinatedEngine: &fakeCoordinatedEngine{}, + } + // A sharded node gets the ShardFailoverManager wrapper, as the real factory + // builds it. + if cfg.sharded { + f.legacyEngine = newTestShardFailoverManager(t, &mockEngine{}) + } + + f.h = &eventHandler{ + lggr: lggr, + capRegistry: registry, + engineRegistry: NewEngineRegistry(), + featureFlags: &v2.EngineFeatureFlags{CoordinatedEngine: limits.NewGateLimiter(cfg.flagOpen)}, + tracer: noop.NewTracerProvider().Tracer(""), + engineFactory: func(ctx context.Context, wfid, _ string, _ types.WorkflowName, _ string, _, _ []byte, _ string, initDone chan<- error) (v2.WorkflowEngine, error) { + f.factoryCalls.Add(1) + if f.factoryErr != nil { + return nil, f.factoryErr + } + initDone <- nil + if f.h.useCoordinatedEngine(ctx, wfid) { + return f.coordinatedEngine, nil + } + return f.legacyEngine, nil + }, + } + if cfg.withCoordinator { + f.coordinator = &recordingCoordinator{} + f.h.triggerCoordinator = f.coordinator + } + if cfg.sharded { + f.h.shardingEnabled = true + f.h.dispatcher = newFakeDispatcher() + } + return f +} + +// newTestWorkflowSpec returns a spec whose workflow ID matches its artifacts, +// so it passes prepareEngineInputs. +func newTestWorkflowSpec(t *testing.T) (*job.WorkflowSpec, types.WorkflowID) { + t.Helper() + owner := []byte{0x01, 0x02, 0x03} + name := "engine-create-test" + binary := []byte("binary") + config := []byte("config") + id, err := pkgworkflows.GenerateWorkflowID(owner, name, binary, config, "") + require.NoError(t, err) + wid := types.WorkflowID(id) + return &job.WorkflowSpec{ + Workflow: hex.EncodeToString(binary), + Config: string(config), + WorkflowID: wid.Hex(), + WorkflowOwner: hex.EncodeToString(owner), + WorkflowName: name, + }, wid +} + +func Test_tryEngineCreate_routing(t *testing.T) { + t.Parallel() + + t.Run("flag open with a coordinator takes the coordinated path", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: true, withCoordinator: true}) + spec, wid := newTestWorkflowSpec(t) + + require.NoError(t, f.h.tryEngineCreate(t.Context(), spec, "TestSource")) + + require.Equal(t, int32(1), f.factoryCalls.Load()) + entry, ok := f.h.engineRegistry.Get(wid) + require.True(t, ok) + require.True(t, entry.Coordinated()) + require.NotEmpty(t, entry.ReconcileKey) + require.Equal(t, []v2.Subscriber{f.coordinatedEngine}, f.coordinator.registered) + }) + + t.Run("flag closed takes the legacy path", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: false, withCoordinator: true}) + spec, wid := newTestWorkflowSpec(t) + + require.NoError(t, f.h.tryEngineCreate(t.Context(), spec, "TestSource")) + + require.Equal(t, int32(1), f.factoryCalls.Load()) + entry, ok := f.h.engineRegistry.Get(wid) + require.True(t, ok) + require.False(t, entry.Coordinated()) + require.Empty(t, f.coordinator.registered) + }) + + t.Run("flag open without a coordinator takes the legacy path", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: true, withCoordinator: false}) + spec, wid := newTestWorkflowSpec(t) + + require.NoError(t, f.h.tryEngineCreate(t.Context(), spec, "TestSource")) + + entry, ok := f.h.engineRegistry.Get(wid) + require.True(t, ok) + require.False(t, entry.Coordinated()) + }) + + t.Run("a sharded node stays on the legacy path with the flag open", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: true, withCoordinator: true, sharded: true}) + spec, wid := newTestWorkflowSpec(t) + + require.NoError(t, f.h.tryEngineCreate(t.Context(), spec, "TestSource")) + + entry, ok := f.h.engineRegistry.Get(wid) + require.True(t, ok) + require.IsType(t, &ShardFailoverManager{}, entry.Service) + require.False(t, entry.Coordinated(), "a sharded engine must never classify as coordinated") + require.Empty(t, f.coordinator.registered, "the coordinator must not be used on a sharded node") + require.Equal(t, engineCounts{legacy: 1}, countEngines(f.h.engineRegistry.GetAll())) + }) + + t.Run("RegisterTriggers failure removes and closes the coordinated engine", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{flagOpen: true, withCoordinator: true}) + f.coordinator.registerErr = errors.New("boom") + spec, wid := newTestWorkflowSpec(t) + + err := f.h.tryEngineCreate(t.Context(), spec, "TestSource") + + require.ErrorContains(t, err, "failed to register triggers via coordinator") + require.False(t, f.h.engineRegistry.Contains(wid)) + require.True(t, f.coordinatedEngine.closed.Load()) + }) + + t.Run("a factory failure is reported and registers nothing", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{withCoordinator: true}) + f.factoryErr = errors.New("boom") + spec, wid := newTestWorkflowSpec(t) + + err := f.h.tryEngineCreate(t.Context(), spec, "TestSource") + + require.ErrorContains(t, err, "failed to create workflow engine") + require.False(t, f.h.engineRegistry.Contains(wid)) + }) + + t.Run("invalid spec is rejected before any engine is built", func(t *testing.T) { + t.Parallel() + f := newEngineCreateFixture(t, nodeConfig{withCoordinator: true}) + spec, _ := newTestWorkflowSpec(t) + spec.WorkflowID = types.WorkflowID{0xff}.Hex() + + err := f.h.tryEngineCreate(t.Context(), spec, "TestSource") + + require.ErrorContains(t, err, "workflowID mismatch") + require.Equal(t, int32(0), f.factoryCalls.Load()) + }) +} + +// Test_useCoordinatedEngine covers the routing policy the engine factory reads. +func Test_useCoordinatedEngine(t *testing.T) { + t.Parallel() + + newHandler := func(flagOpen, withCoordinator, sharded bool) *eventHandler { + h := &eventHandler{ + lggr: logger.Test(t), + featureFlags: &v2.EngineFeatureFlags{CoordinatedEngine: limits.NewGateLimiter(flagOpen)}, + } + if withCoordinator { + h.triggerCoordinator = &recordingCoordinator{} + } + if sharded { + h.shardingEnabled = true + h.dispatcher = newFakeDispatcher() + } + return h + } + + t.Run("flag open with a coordinator", func(t *testing.T) { + t.Parallel() + require.True(t, newHandler(true, true, false).useCoordinatedEngine(t.Context(), "wfid")) + }) + + t.Run("flag closed", func(t *testing.T) { + t.Parallel() + require.False(t, newHandler(false, true, false).useCoordinatedEngine(t.Context(), "wfid")) + }) + + t.Run("no coordinator wired", func(t *testing.T) { + t.Parallel() + require.False(t, newHandler(true, false, false).useCoordinatedEngine(t.Context(), "wfid")) + }) + + t.Run("sharded nodes stay on the legacy engine", func(t *testing.T) { + t.Parallel() + require.False(t, newHandler(true, true, true).useCoordinatedEngine(t.Context(), "wfid")) + }) + + t.Run("nil feature flags", func(t *testing.T) { + t.Parallel() + h := newHandler(true, true, false) + h.featureFlags = nil + require.False(t, h.useCoordinatedEngine(t.Context(), "wfid")) + }) +} diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index 4943d276db7..59146bafca7 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -65,6 +65,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncerlimiter" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2/triggers" ) type mockFetchResp struct { @@ -125,6 +126,22 @@ func (m *mockEngine) HealthReport() map[string]error { return nil } func (m *mockEngine) Name() string { return "mockEngine" } +func (m *mockEngine) ExecuteTrigger(context.Context, triggers.CoordinatedEvent) error { return nil } + +func (m *mockEngine) Subscribe(context.Context) ([]*sdk.TriggerSubscription, error) { + return nil, nil +} + +func (m *mockEngine) Tenant() contexts.CRE { return contexts.CRE{} } + +func (m *mockEngine) Drain() bool { return false } + +func (m *mockEngine) ActiveExecutions() int32 { return 0 } + +func (m *mockEngine) DrainStartedAt() (time.Time, bool) { return time.Time{}, false } + +func (m *mockEngine) IsCoordinated() bool { return false } + type mockDrainableEngine struct { mockEngine draining atomic.Bool @@ -162,7 +179,7 @@ func (m *mockDrainableEngine) Close() error { // mockEngineFactory returns a standard mock engine factory for tests. // It sends nil to initDone to signal successful initialization. -func mockEngineFactory(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) { +func mockEngineFactory(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) { if initDone != nil { initDone <- nil } @@ -317,7 +334,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { signedConfigURL: {Body: config, Err: nil}, }) }, - engineFactoryFn: func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) { + engineFactoryFn: func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) { if _, err := hex.DecodeString(name.Hex()); err != nil { return nil, fmt.Errorf("invalid workflow name: %w", err) } @@ -361,7 +378,7 @@ func Test_workflowRegisteredHandler(t *testing.T) { signedConfigURL: {Body: config, Err: nil}, }) }, - engineFactoryFn: func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) { + engineFactoryFn: func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) { if initDone != nil { initDone <- nil } @@ -960,7 +977,7 @@ type testCase struct { fetcherFactory func(wfID []byte) *mockFetcher Event func(wfID []byte, wfName string, wfOwner []byte) WorkflowRegisteredEvent validationFn func(t *testing.T, ctx context.Context, event WorkflowRegisteredEvent, h *eventHandler, s *artifacts.Store, wfOwner []byte, wfName string, wfID types.WorkflowID, fetcher *mockFetcher, binaryURL, configURL string) - engineFactoryFn func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (services.Service, error) + engineFactoryFn func(ctx context.Context, wfid, owner string, name types.WorkflowName, tag string, config, binary []byte, binaryURL string, initDone chan<- error) (v2.WorkflowEngine, error) } func testRunningWorkflow(t *testing.T, tc testCase) { diff --git a/core/services/workflows/syncer/v2/metrics.go b/core/services/workflows/syncer/v2/metrics.go index 9485daf2651..76f22eb9978 100644 --- a/core/services/workflows/syncer/v2/metrics.go +++ b/core/services/workflows/syncer/v2/metrics.go @@ -11,16 +11,27 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/beholder" ) +// engineType label values for the running-engines gauge. +const ( + engineTypeCoordinated = "coordinated" + engineTypeLegacy = "legacy" +) + type metrics struct { handleDuration metric.Int64Histogram fetchedWorkflows metric.Int64Gauge runningWorkflows metric.Int64Gauge drainingWorkflows metric.Int64Gauge - completedSyncs metric.Int64Counter - drainStarted metric.Int64Counter - drainCompleted metric.Int64Counter - drainDuration metric.Int64Histogram - deleteDeferred metric.Int64Counter + // runningEngines counts the engines currently in the registry, split by + // engineType ("coordinated" or "legacy"). Both series are always reported, + // including the zeroes, so an absent series means the node stopped + // reporting rather than "no engines of that type". + runningEngines metric.Int64Gauge + completedSyncs metric.Int64Counter + drainStarted metric.Int64Counter + drainCompleted metric.Int64Counter + drainDuration metric.Int64Histogram + deleteDeferred metric.Int64Counter // Per-source metrics for multi-source observability sourceHealth metric.Int64Gauge // 1=healthy, 0=unhealthy per source @@ -62,6 +73,19 @@ func (m *metrics) recordDrainingWorkflows(ctx context.Context, count int) { m.drainingWorkflows.Record(ctx, int64(count)) } +// recordRunningEngines reports the engine-type split of the registry. Called +// once per reconciliation tick, with both counts, so the two series stay in +// lockstep and "coordinated == 0" is an assertion the alert can rely on rather +// than the absence of a sample. +func (m *metrics) recordRunningEngines(ctx context.Context, coordinated, legacy int) { + m.runningEngines.Record(ctx, int64(coordinated), metric.WithAttributes( + attribute.String("engineType", engineTypeCoordinated), + )) + m.runningEngines.Record(ctx, int64(legacy), metric.WithAttributes( + attribute.String("engineType", engineTypeLegacy), + )) +} + func (m *metrics) incrementCompletedSyncs(ctx context.Context) { m.completedSyncs.Add(ctx, 1) } @@ -288,6 +312,11 @@ func newMetrics() (*metrics, error) { return nil, err } + runningEngines, err := beholder.GetMeter().Int64Gauge("platform_workflow_registry_syncer_running_engines") + if err != nil { + return nil, err + } + completedSyncs, err := beholder.GetMeter().Int64Counter("platform_workflow_registry_syncer_completed_syncs_total") if err != nil { return nil, err @@ -374,6 +403,7 @@ func newMetrics() (*metrics, error) { fetchedWorkflows: fetchedWorkflows, runningWorkflows: runningWorkflows, drainingWorkflows: drainingWorkflows, + runningEngines: runningEngines, completedSyncs: completedSyncs, drainStarted: drainStarted, drainCompleted: drainCompleted, diff --git a/core/services/workflows/syncer/v2/shard_failover_manager.go b/core/services/workflows/syncer/v2/shard_failover_manager.go index 987f359816e..dcf2b5116c8 100644 --- a/core/services/workflows/syncer/v2/shard_failover_manager.go +++ b/core/services/workflows/syncer/v2/shard_failover_manager.go @@ -8,9 +8,11 @@ import ( "time" commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" ringpb "github.com/smartcontractkit/chainlink-protos/ring/go" "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/sharding" @@ -25,6 +27,11 @@ import ( // cachedExpiry is how long a cached trigger event is kept for potential failover replay. const cachedExpiry = 10 * time.Minute +// ErrCoordinatedShardingUnsupported is returned by the coordinator-facing +// methods of ShardFailoverManager. Sharding and the coordinated engine are +// currently not compatible, so nothing should reach them. +var ErrCoordinatedShardingUnsupported = errors.New("coordinated engine is not supported on a sharded node") + // ShardFailoverManager wraps a workflow Engine and handles shard ownership // decisions externally, keeping the Engine oblivious to sharding. On the // primary shard it allows all trigger events through. On a secondary shard @@ -409,4 +416,29 @@ func (m *ShardFailoverManager) ActiveExecutions() int32 { return m.engine.Active // DrainStartedAt delegates to the wrapped engine. func (m *ShardFailoverManager) DrainStartedAt() (time.Time, bool) { return m.engine.DrainStartedAt() } -var _ DrainableService = (*ShardFailoverManager)(nil) +// ExecuteTrigger is unreachable on a sharded node: the coordinator is the only +// caller of the EventSink path, and a sharded node never takes the coordinated +// path. Failover replay calls the wrapped engine directly, not this method. +func (m *ShardFailoverManager) ExecuteTrigger(context.Context, triggers.CoordinatedEvent) error { + return ErrCoordinatedShardingUnsupported +} + +// Subscribe is unreachable on a sharded node: only the coordinator calls it, +// while registering triggers for a coordinated engine. +func (m *ShardFailoverManager) Subscribe(context.Context) ([]*sdkpb.TriggerSubscription, error) { + return nil, ErrCoordinatedShardingUnsupported +} + +// Tenant delegates to the wrapped engine. It has no error to return, and as a +// plain getter it cannot half-work, so it stays a passthrough. +func (m *ShardFailoverManager) Tenant() contexts.CRE { return m.engine.Tenant() } + +// IsCoordinated delegates to the wrapped engine. Unlike the methods above this +// is called on the legacy path — every reconciliation tick reads it to classify +// the engine — so it must answer rather than fail. +func (m *ShardFailoverManager) IsCoordinated() bool { return m.engine.IsCoordinated() } + +var ( + _ DrainableService = (*ShardFailoverManager)(nil) + _ v2.WorkflowEngine = (*ShardFailoverManager)(nil) +) diff --git a/core/services/workflows/syncer/v2/shard_failover_manager_test.go b/core/services/workflows/syncer/v2/shard_failover_manager_test.go index fb73d29a82d..eee0d0f8bf0 100644 --- a/core/services/workflows/syncer/v2/shard_failover_manager_test.go +++ b/core/services/workflows/syncer/v2/shard_failover_manager_test.go @@ -19,6 +19,8 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/sharding" remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types" p2ptypes "github.com/smartcontractkit/chainlink/v2/core/services/p2p/types" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2/triggers" ) // --- test doubles --- @@ -340,3 +342,49 @@ func (r *noopReceiver) Receive(context.Context, *remotetypes.MessageBody) {} // Ensure fakeDonNotifier satisfies the interface. var _ capabilities.DonSubscriber = (*fakeDonSubscriber)(nil) + +// newTestShardFailoverManager wraps engine in a manager that is never started, +// so the sharding collaborators are left nil. +func newTestShardFailoverManager(t *testing.T, engine v2.WorkflowEngine) *ShardFailoverManager { + t.Helper() + m := NewShardFailoverManager(ShardFailoverManagerConfig{ + ShardingEnabled: true, + WorkflowID: "wf-1", + WorkflowOwner: "0xowner", + Logger: logger.Test(t), + }) + m.SetEngine(engine) + return m +} + +// TestShardFailoverManager_DelegatesIsCoordinated covers the classification of +// the wrapper a sharded node registers, which must follow the engine it wraps. +func TestShardFailoverManager_DelegatesIsCoordinated(t *testing.T) { + t.Parallel() + + t.Run("wrapping a legacy engine reports legacy", func(t *testing.T) { + t.Parallel() + m := newTestShardFailoverManager(t, &mockEngine{}) + require.False(t, m.IsCoordinated()) + }) + + // Unreachable while sharding and the coordinated engine are mutually + // exclusive, but asserted so the value stays delegated rather than assumed. + t.Run("wrapping a coordinated engine reports coordinated", func(t *testing.T) { + t.Parallel() + m := newTestShardFailoverManager(t, &fakeCoordinatedDrainableEngine{}) + require.True(t, m.IsCoordinated()) + }) +} + +// TestShardFailoverManager_CoordinatorMethodsUnsupported covers the +// coordinator-facing methods, which must fail rather than delegate until +// sharding and the coordinated engine can work together. +func TestShardFailoverManager_CoordinatorMethodsUnsupported(t *testing.T) { + t.Parallel() + m := newTestShardFailoverManager(t, &mockEngine{}) + + _, err := m.Subscribe(t.Context()) + require.ErrorIs(t, err, ErrCoordinatedShardingUnsupported) + require.ErrorIs(t, m.ExecuteTrigger(t.Context(), triggers.CoordinatedEvent{}), ErrCoordinatedShardingUnsupported) +} diff --git a/core/services/workflows/syncer/v2/trigger_coordinator.go b/core/services/workflows/syncer/v2/trigger_coordinator.go new file mode 100644 index 00000000000..a4025e9b1e5 --- /dev/null +++ b/core/services/workflows/syncer/v2/trigger_coordinator.go @@ -0,0 +1,103 @@ +package v2 + +import ( + "context" + "errors" + + "github.com/jonboulle/clockwork" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" +) + +// ErrWorkflowNotCoordinated is returned by UnregisterTriggers for +// a workflowID the coordinator never registered. +var ErrWorkflowNotCoordinated = errors.New("workflow not registered with the trigger coordinator") + +// RegistrationParams is the per-workflow metadata RegisterTriggers needs to +// build each capability's RequestMetadata. +type RegistrationParams struct { + WorkflowOwner string + WorkflowName string + DecodedWorkflowName string + WorkflowTag string + WorkflowDonID uint32 + WorkflowDonConfigVersion uint32 + WorkflowRegistryChainSelector string + WorkflowRegistryAddress string +} + +// TriggerCoordinator owns trigger registration, the trigger handle map, event +// delivery, and acknowledgement for every workflow running the coordinated engine. +// It is a node-level singleton, started and stopped with the syncer. +type TriggerCoordinator interface { + services.Service + v2.Acknowledger + + // RegisterTriggers calls subscriber.Subscribe to obtain the engine's trigger + // subscriptions, registers them with the capability registry, retains the + // resulting handles and reader goroutines, and returns the registered + // trigger capability IDs. On any failure it unregisters what it already + // registered for this call and returns the error. + // Partial registration is never left behind. + RegisterTriggers(ctx context.Context, subscriber v2.Subscriber, params RegistrationParams) ([]string, error) + + // UnregisterTriggers stops ingress for workflowID immediately (unregisters with the capability registry) + // and cleans up the handle map once the engine has been drained and closed, so an execution already in flight + // can still resolve its handle to ACK. It also ensures that any resources associated with the workflow are properly released. + // This includes managing the workflowLimits by calling workflowLimits.Free. + // + // Returns ErrWorkflowNotCoordinated if workflowID was never registered here + UnregisterTriggers(workflowID string) error +} + +type noopTriggerCoordinator struct { + services.Service + eng *services.Engine + lggr logger.Logger +} + +// NewTriggerCoordinator returns the no-op coordinator: it logs the registration +// and teardown calls the syncer makes, and does nothing else. No trigger is +// registered with the capability registry and no event is ever delivered, so a +// workflow routed to the coordinated engine while this implementation is in +// place runs but never fires. +// +// capReg, engineRegistry and clock are unused here on purpose — the signature is +// the one the real coordinator needs, so wiring it does not churn call sites. +func NewTriggerCoordinator(capReg registry.CapabilitiesRegistry, engineRegistry *EngineRegistry, clock clockwork.Clock, lggr logger.Logger) TriggerCoordinator { + c := &noopTriggerCoordinator{ + lggr: logger.Named(lggr, "TriggerCoordinator"), + } + + c.Service, c.eng = services.Config{ + Name: "TriggerCoordinator", + Start: func(context.Context) error { + c.lggr.Warnw("No-op trigger coordinator started: workflows on the coordinated engine will register no triggers and receive no events") + return nil + }, + Close: func() error { return nil }, + }.NewServiceEngine(c.lggr) + + return c +} + +func (c *noopTriggerCoordinator) RegisterTriggers(ctx context.Context, subscriber v2.Subscriber, params RegistrationParams) ([]string, error) { + c.lggr.Infow("No-op RegisterTriggers", + "workflowID", subscriber.Tenant().Workflow, + "workflowOwner", params.WorkflowOwner, + "workflowName", params.DecodedWorkflowName, + "workflowDonID", params.WorkflowDonID) + return []string{}, nil +} + +func (c *noopTriggerCoordinator) Ack(ctx context.Context, triggerCapID, triggerRegistrationID, eventID string) error { + return nil +} + +func (c *noopTriggerCoordinator) UnregisterTriggers(workflowID string) error { + c.lggr.Infow("No-op UnregisterTriggers", "workflowID", workflowID) + return nil +} diff --git a/core/services/workflows/syncer/v2/workflow_registry.go b/core/services/workflows/syncer/v2/workflow_registry.go index 56143644606..d45ffc9a616 100644 --- a/core/services/workflows/syncer/v2/workflow_registry.go +++ b/core/services/workflows/syncer/v2/workflow_registry.go @@ -1075,22 +1075,42 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context) runningWorkflows := w.engineRegistry.GetAll() w.metrics.recordRunningWorkflows(ctx, len(runningWorkflows)) - drainingWorkflows := 0 - for _, workflow := range runningWorkflows { - drainable, isDrainable := workflow.Service.(DrainableService) - if !isDrainable { - continue - } - if _, draining := drainable.DrainStartedAt(); draining { - drainingWorkflows++ - } - } - w.metrics.recordDrainingWorkflows(ctx, drainingWorkflows) + counts := countEngines(runningWorkflows) + w.metrics.recordDrainingWorkflows(ctx, counts.draining) + w.metrics.recordRunningEngines(ctx, counts.coordinated, counts.legacy) w.metrics.incrementCompletedSyncs(ctx) } } } +// engineCounts is the per-tick breakdown of the engines in the registry. +// coordinated + legacy always equals the number of engines; draining overlaps +// with both. +type engineCounts struct { + draining int + coordinated int + legacy int +} + +func countEngines(engines []ServiceWithMetadata) engineCounts { + var c engineCounts + for _, e := range engines { + if e.Coordinated() { + c.coordinated++ + } else { + c.legacy++ + } + drainable, isDrainable := e.Service.(DrainableService) + if !isDrainable { + continue + } + if _, draining := drainable.DrainStartedAt(); draining { + c.draining++ + } + } + return c +} + // reconcileOrphanedSpecs releases persisted specs whose workflow ID is absent // from all sources' metadata, catching workflows deleted while the node was // down — no engine exists at the next tick, so engine reconciliation can't diff --git a/core/services/workflows/syncer/v2/workflow_syncer_v2_test.go b/core/services/workflows/syncer/v2/workflow_syncer_v2_test.go index f6ad2d7d06c..e8b0d41be16 100644 --- a/core/services/workflows/syncer/v2/workflow_syncer_v2_test.go +++ b/core/services/workflows/syncer/v2/workflow_syncer_v2_test.go @@ -413,7 +413,7 @@ func Test_RegistrySyncer_WorkflowRegistered_InitiallyActivatedV2(t *testing.T) { require.NoError(t, err) handler, err := NewEventHandler(lggr, wfStore, nil, true, capRegistry, &confidentialrelay.ExecutionHandlers{}, er, - emitter, limiters, featureFlags, rl, wl, store, workflowEncryptionKey, donNotifier, WithStaticEngine(&mockService{}), withTestOrgResolver()) + emitter, limiters, featureFlags, rl, wl, store, workflowEncryptionKey, donNotifier, WithStaticEngine(&mockEngine{}), withTestOrgResolver()) require.NoError(t, err) worker, err := NewWorkflowRegistry( diff --git a/core/services/workflows/v2/config.go b/core/services/workflows/v2/config.go index bf993076a24..8470792fbee 100644 --- a/core/services/workflows/v2/config.go +++ b/core/services/workflows/v2/config.go @@ -12,7 +12,6 @@ import ( commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry" "github.com/smartcontractkit/chainlink-common/pkg/config" - "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/custmsg" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" @@ -387,6 +386,10 @@ type EngineFeatureFlags struct { // which lets ops schedule a healing window across the DON via cresettings. // Nil when construction fails; call sites must nil-check. WorkflowTagBackfill limits.RangeLimiter[config.Timestamp] + + // CoordinatedEngine selects the coordinatedEngine over the legacy trigger-owning + // Engine for newly created workflows. Nil when construction fails; call sites must nil-check. + CoordinatedEngine limits.GateLimiter } func NewFeatureFlags(lf limits.Factory, cfgFn func(*cresettings.Workflows)) (*EngineFeatureFlags, error) { @@ -398,8 +401,14 @@ func NewFeatureFlags(lf limits.Factory, cfgFn func(*cresettings.Workflows)) (*En if err != nil { return nil, fmt.Errorf("workflow tag backfill flag: %w", err) } + + coordinatedEngine, err := limits.MakeGateLimiter(lf, cresettings.Default.CoordinatedEngineEnabled) + if err != nil { + return nil, fmt.Errorf("coordinated engine flag: %w", err) + } return &EngineFeatureFlags{ WorkflowTagBackfill: workflowTagBackfill, + CoordinatedEngine: coordinatedEngine, }, nil } @@ -427,12 +436,6 @@ type LifecycleHooks struct { // has completed initialization. It is also helpful for testing. OnInitialized func(err error) - // OnSubscriptionsReady is called after the WASM Subscribe call returns - // and the subscriptions have been validated, but before trigger - // registration begins. It allows the caller (syncer/dispatcher) to - // inspect or modify the subscriptions before they are registered with - // the capabilities registry. Returning an error aborts initialization. - OnSubscriptionsReady func(subs []*sdkpb.TriggerSubscription, cre contexts.CRE) error OnSubscribedToTriggers func(triggerIDs []string) OnTriggerEventDropped func(triggerID, eventID, reason string) OnExecutionFinished func(executionID string, status string) @@ -522,9 +525,6 @@ func (h *LifecycleHooks) setDefaultHooks() { if h.OnInitialized == nil { h.OnInitialized = func(err error) {} } - if h.OnSubscriptionsReady == nil { - h.OnSubscriptionsReady = func(subs []*sdkpb.TriggerSubscription, cre contexts.CRE) error { return nil } - } if h.OnSubscribedToTriggers == nil { h.OnSubscribedToTriggers = func(triggerIDs []string) {} } diff --git a/core/services/workflows/v2/engine_test.go b/core/services/workflows/v2/engine_test.go index d15b3eb0991..3c0c417cfb2 100644 --- a/core/services/workflows/v2/engine_test.go +++ b/core/services/workflows/v2/engine_test.go @@ -2900,10 +2900,6 @@ func newTestEngine( OnInitialized: func(err error) { e.initializedCh <- err }, - OnSubscriptionsReady: func(_ []*sdkpb.TriggerSubscription, _ contexts.CRE) error { - e.subscriptionsReadyCalls.Add(1) - return nil - }, OnSubscribedToTriggers: func(triggerIDs []string) { e.subscribedToTriggersCh <- triggerIDs }, diff --git a/deployment/go.mod b/deployment/go.mod index a5c18580870..57475d24166 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -46,11 +46,11 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d - github.com/smartcontractkit/chainlink-data-streams v1.1.1 + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260916140808-f5d22b3cb3ee diff --git a/deployment/go.sum b/deployment/go.sum index 524ce8bf8e5..520769a748d 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1403,18 +1403,18 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/docs/CONFIG.md b/docs/CONFIG.md index f25b19bb1ee..9c602d1dfa3 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -9228,6 +9228,126 @@ AcceptanceTimeout = '30s'

+
Tempo Mainnet (4217)

+ +```toml +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 50 +SafeDepth = 0 +FinalityTagEnabled = true +SafeTagSupported = true +LogBackfillBatchSize = 1000 +LogPollInterval = '1s' +LogPollerSkipEmptyBlocks = false +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 1 +MinContractPayment = '0.00001 link' +NonceAutoSync = true +NoNewHeadsThreshold = '3m0s' +LogBroadcasterEnabled = false +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 2 +FinalizedBlockOffset = 0 +NoNewFinalizedHeadsThreshold = '0s' + +[Transactions] +Enabled = true +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '1m0s' +ConfirmationTimeout = '1m0s' + +[Transactions.AutoPurge] +Enabled = false + +[Transactions.TransactionManagerV2] +Enabled = false + +[BalanceMonitor] +Enabled = true +ERC20TokenAddress = '0x20C0000000000000000000000000000000000000' + +[GasEstimator] +Mode = 'FeeHistory' +PriceDefault = '1 gwei' +PriceMax = '1 micro' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '1' +LimitTransfer = 21000 +EstimateLimit = false +BumpMin = '5 gwei' +BumpPercent = 20 +BumpThreshold = 30 +EIP1559DynamicFees = false +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 8 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 60 + +[GasEstimator.FeeHistory] +CacheTimeout = '4s' + +[HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = false +PersistenceEnabled = false +PersistenceBatchSize = 100 + +[NodePool] +PollFailureThreshold = 5 +PollSuccessThreshold = 0 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 15 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' +FinalizedStateCheckFailureThreshold = 0 +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' +NewHeadsPollInterval = '1s' +VerifyChainID = true +ExternalRequestMaxResponseSize = 1000000 + +[NodePool.Errors] +FinalizedStateUnavailable = '(: |^)(missing trie node|state not available|historical state unavailable)' + +[OCR] +ContractConfirmations = 1 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[Workflow] +GasLimitDefault = 400000 +TxAcceptanceState = 2 +PollPeriod = '2s' +AcceptanceTimeout = '30s' +``` + +

+
Worldchain Testnet (4801)

```toml @@ -10832,6 +10952,126 @@ AcceptanceTimeout = '30s'

+
Tempo Testnet Moderato (42431)

+ +```toml +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 50 +SafeDepth = 0 +FinalityTagEnabled = true +SafeTagSupported = true +LogBackfillBatchSize = 1000 +LogPollInterval = '1s' +LogPollerSkipEmptyBlocks = false +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 1 +MinContractPayment = '0.00001 link' +NonceAutoSync = true +NoNewHeadsThreshold = '3m0s' +LogBroadcasterEnabled = false +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 2 +FinalizedBlockOffset = 0 +NoNewFinalizedHeadsThreshold = '0s' + +[Transactions] +Enabled = true +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '1m0s' +ConfirmationTimeout = '1m0s' + +[Transactions.AutoPurge] +Enabled = false + +[Transactions.TransactionManagerV2] +Enabled = false + +[BalanceMonitor] +Enabled = true +ERC20TokenAddress = '0x20C0000000000000000000000000000000000000' + +[GasEstimator] +Mode = 'FeeHistory' +PriceDefault = '1 gwei' +PriceMax = '1 micro' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '4' +LimitTransfer = 21000 +EstimateLimit = false +BumpMin = '5 gwei' +BumpPercent = 20 +BumpThreshold = 30 +EIP1559DynamicFees = false +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 8 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 60 + +[GasEstimator.FeeHistory] +CacheTimeout = '4s' + +[HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = false +PersistenceEnabled = false +PersistenceBatchSize = 100 + +[NodePool] +PollFailureThreshold = 5 +PollSuccessThreshold = 0 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 15 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' +HistoricalBalanceCheckAddress = '0x0000000000000000000000000000000000000000' +FinalizedStateCheckFailureThreshold = 0 +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' +NewHeadsPollInterval = '1s' +VerifyChainID = true +ExternalRequestMaxResponseSize = 1000000 + +[NodePool.Errors] +FinalizedStateUnavailable = '(: |^)(missing trie node|state not available|historical state unavailable)' + +[OCR] +ContractConfirmations = 1 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[Workflow] +GasLimitDefault = 400000 +TxAcceptanceState = 2 +PollPeriod = '2s' +AcceptanceTimeout = '30s' +``` + +

+
Hemi Mainnet (43111)

```toml @@ -16403,6 +16643,7 @@ FeeBoost enables node to immediately boost txs to the max gas price configured ```toml [EVM.BalanceMonitor] Enabled = true # Default +ERC20TokenAddress = "0x20C0000000000000000000000000000000000000" # Example ``` @@ -16412,6 +16653,12 @@ Enabled = true # Default ``` Enabled balance monitoring for all keys. +### ERC20TokenAddress +```toml +ERC20TokenAddress = "0x20C0000000000000000000000000000000000000" # Example +``` +ERC20TokenAddress is the address of an ERC-20 type token that a chain supports as a fee token. + ## EVM.GasEstimator ```toml [EVM.GasEstimator] diff --git a/go.md b/go.md index 45e9aaf0b6a..a645767a478 100644 --- a/go.md +++ b/go.md @@ -593,7 +593,7 @@ flowchart LR chainlink/system-tests/tests/smoke/cre/stellar/stellarwrite --> cre-sdk-go/capabilities/blockchain/stellar chainlink/system-tests/tests/smoke/cre/stellar/stellarwrite --> cre-sdk-go/capabilities/scheduler/cron click chainlink/system-tests/tests/smoke/cre/stellar/stellarwrite href "https://github.com/smartcontractkit/chainlink" - chainlink/system-tests/tests/smoke/cre/vaultsecret --> cre-sdk-go/capabilities/scheduler/cron + chainlink/system-tests/tests/smoke/cre/vaultsecret --> cre-sdk-go/capabilities/networking/http click chainlink/system-tests/tests/smoke/cre/vaultsecret href "https://github.com/smartcontractkit/chainlink" chainlink/v2 --> chainlink-aptos/codec chainlink/v2 --> chainlink-ccv diff --git a/go.mod b/go.mod index 73c4825a890..87185644328 100644 --- a/go.mod +++ b/go.mod @@ -82,11 +82,11 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 - github.com/smartcontractkit/chainlink-data-streams v1.1.1 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260710181111-6417709a55ee github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 diff --git a/go.sum b/go.sum index e067dafc04d..e891a1bb9d3 100644 --- a/go.sum +++ b/go.sum @@ -1104,16 +1104,16 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62/go.mod h1:c++Ed0jeYn4w8WIfiAboRwN6ClRRVGglL0qwB/QjJvM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260710181111-6417709a55ee h1:MfUoOYDYeE4vzk2FXDw65V0CjDlVczYd5ohCBRwByjk= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index f07c1d098d7..78edd2fd3ec 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -34,10 +34,10 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea github.com/smartcontractkit/chainlink-sui v0.0.0-20260922143238-46cc062ae5a3 @@ -410,7 +410,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 8b7eceeeb22..03415ae1343 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1392,18 +1392,18 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 6f29b5d19f7..db655a5b95e 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -24,9 +24,9 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.8 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.5 github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5 @@ -481,7 +481,7 @@ require ( github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 0848f18e14f..e2b72f32542 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1630,18 +1630,18 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/plugins/plugins.public.yaml b/plugins/plugins.public.yaml index 2a2fdbaa244..06ea33bfaa6 100644 --- a/plugins/plugins.public.yaml +++ b/plugins/plugins.public.yaml @@ -43,7 +43,7 @@ plugins: streams: - moduleURI: "github.com/smartcontractkit/chainlink-data-streams" - gitRef: "v1.1.1" + gitRef: "v1.1.2-0.20260929142841-14ba41fe8d23" installPath: "./mercury/cmd/chainlink-mercury" ton: @@ -58,7 +58,7 @@ plugins: evm: - moduleURI: "github.com/smartcontractkit/chainlink-evm" - gitRef: "v0.3.4-0.20260917132551-2a2b38e8b744" + gitRef: "v0.3.4-0.20260929161505-cc645fce4250" installPath: "./pkg/cmd/chainlink-evm" capability-cron: @@ -69,7 +69,7 @@ plugins: capability-consensus: - moduleURI: "github.com/smartcontractkit/capabilities/consensus" - gitRef: "2aaae97ddf6900b082391c4756687b7584bfaf8a" + gitRef: "968e708eec352f300465080cbbec49dff36aa488" installPath: "." capability-httpaction: diff --git a/system-tests/lib/cre/workflow/compile.go b/system-tests/lib/cre/workflow/compile.go index fea3d162ac6..443f598e7d4 100644 --- a/system-tests/lib/cre/workflow/compile.go +++ b/system-tests/lib/cre/workflow/compile.go @@ -3,15 +3,22 @@ package workflow import ( "bytes" "context" + "crypto/sha256" "encoding/base64" + "encoding/hex" "fmt" + "io/fs" "os" "os/exec" "path/filepath" + "sort" "strings" + "sync" "github.com/andybalholm/brotli" "github.com/pkg/errors" + + "github.com/smartcontractkit/chainlink-testing-framework/framework" ) type Language = string @@ -21,6 +28,9 @@ const ( LanguageTS Language = "typescript" ) +// Set CRE_TEST_COMPILE_CACHE_DISABLED=true to bypass the in-process compile cache. +const compileCacheDisabledEnv = "CRE_TEST_COMPILE_CACHE_DISABLED" + // CompileWorkflow compiles a workflow from a file path (absolute or relative) and returns the path to the compiled workflow. // workflowFilePath is the path to the workflow file. // workflowName is the name of the workflow. @@ -33,6 +43,8 @@ func CompileWorkflow(ctx context.Context, workflowFilePath, workflowName string) // CompileWorkflowToDir compiles a workflow and stores build artifacts in outputDir. // If outputDir is empty, a temporary directory is created automatically. +// Artifacts are cached in-process by a hash of the workflow directory, so +// identical sources compile once per test run. func CompileWorkflowToDir(ctx context.Context, workflowFilePath, workflowName, outputDir string) (string, error) { if len(workflowName) < 10 { return "", errors.New("workflow name must be at least 10 characters long") @@ -48,6 +60,175 @@ func CompileWorkflowToDir(ctx context.Context, workflowFilePath, workflowName, o return "", errors.Wrap(mkErr, "failed to prepare workflow build dir") } + if compileCacheEnabled() { + key, keyErr := workflowSourceCacheKey(workflowFilePath) + if keyErr == nil { + return compileWorkflowToDirCached(ctx, key, workflowFilePath, workflowName, outputDir) + } + framework.L.Warn().Err(keyErr).Str("workflow_file", workflowFilePath).Msg("failed to hash workflow sources; compiling without the compile cache") + } + + return compileWorkflowToDirUncached(ctx, workflowFilePath, workflowName, outputDir) +} + +type compileCacheEntry struct { + done chan struct{} + artifact []byte + err error +} + +var ( + compileCacheMu sync.Mutex + compileCache = map[string]*compileCacheEntry{} +) + +func compileCacheEnabled() bool { + return strings.TrimSpace(strings.ToLower(os.Getenv(compileCacheDisabledEnv))) != "true" +} + +// getOrCreateCompileCacheEntry returns the entry for key and whether the +// caller is the leader that must compile it. +func getOrCreateCompileCacheEntry(key string) (entry *compileCacheEntry, leader bool) { + compileCacheMu.Lock() + defer compileCacheMu.Unlock() + if existing, ok := compileCache[key]; ok { + return existing, false + } + entry = &compileCacheEntry{done: make(chan struct{})} + compileCache[key] = entry + return entry, true +} + +// finalizeCompileCacheEntry publishes the result to waiters. Failed entries +// are evicted so later callers retry instead of reusing the failure. +func finalizeCompileCacheEntry(key string, entry *compileCacheEntry, artifact []byte, err error) { + entry.artifact = artifact + entry.err = err + compileCacheMu.Lock() + defer compileCacheMu.Unlock() + if err != nil { + delete(compileCache, key) + } + close(entry.done) +} + +func compileWorkflowToDirCached(ctx context.Context, cacheKey, workflowFilePath, workflowName, outputDir string) (string, error) { + for { + entry, leader := getOrCreateCompileCacheEntry(cacheKey) + if leader { + return compileAsCompileCacheLeader(ctx, cacheKey, entry, workflowFilePath, workflowName, outputDir) + } + + if err := waitForCompileCacheEntry(ctx, entry); err != nil { + // The failed entry was evicted, so retrying makes this caller the + // leader (covers a previous leader whose context was cancelled). + framework.L.Warn().Str("workflow_name", workflowName).Err(err).Msg("in-flight workflow compile failed; retrying as the compile leader") + continue + } + + framework.L.Info().Str("workflow_name", workflowName).Str("workflow_file", workflowFilePath).Msg("workflow compile cache hit; reusing compiled workflow artifact") + return writeCompressedArtifact(outputDir, workflowName, entry.artifact) + } +} + +func compileAsCompileCacheLeader(ctx context.Context, cacheKey string, entry *compileCacheEntry, workflowFilePath, workflowName, outputDir string) (string, error) { + framework.L.Info().Str("workflow_name", workflowName).Str("workflow_file", workflowFilePath).Msg("workflow compile cache miss; compiling workflow") + + artifactPath, compileErr := compileWorkflowToDirUncached(ctx, workflowFilePath, workflowName, outputDir) + + var artifact []byte + cacheErr := compileErr + if cacheErr == nil { + b, readErr := os.ReadFile(artifactPath) + if readErr != nil { + // Artifact is still valid for this caller; only caching is skipped. + cacheErr = errors.Wrap(readErr, "failed to read compiled workflow artifact for the compile cache") + } else { + artifact = b + } + } + + finalizeCompileCacheEntry(cacheKey, entry, artifact, cacheErr) + return artifactPath, compileErr +} + +// waitForCompileCacheEntry blocks until the leader finishes or ctx ends. +func waitForCompileCacheEntry(ctx context.Context, entry *compileCacheEntry) error { + select { + case <-entry.done: + if entry.err != nil { + return errors.Wrap(entry.err, "in-flight workflow compile failed") + } + if len(entry.artifact) == 0 { + return errors.New("in-flight workflow compile produced no artifact") + } + return nil + case <-ctx.Done(): + return errors.Wrap(ctx.Err(), "cancelled while waiting for an in-flight workflow compile") + } +} + +// workflowSourceCacheKey hashes all build inputs in the workflow's directory +// (sources, go.mod/go.sum, package.json, ...), excluding build outputs. +func workflowSourceCacheKey(workflowFilePath string) (string, error) { + dir := filepath.Dir(workflowFilePath) + + var files []string + walkErr := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == ".git" || d.Name() == "node_modules" { + return filepath.SkipDir + } + return nil + } + if strings.HasSuffix(d.Name(), ".wasm") || strings.HasSuffix(d.Name(), ".br.b64") { + return nil + } + files = append(files, path) + return nil + }) + if walkErr != nil { + return "", errors.Wrapf(walkErr, "failed to walk workflow source dir %s", dir) + } + + sort.Strings(files) + hasher := sha256.New() + for _, path := range files { + rel, relErr := filepath.Rel(dir, path) + if relErr != nil { + return "", errors.Wrap(relErr, "failed to relativize workflow source path") + } + content, readErr := os.ReadFile(path) + if readErr != nil { + return "", errors.Wrapf(readErr, "failed to read workflow source file %s", path) + } + hasher.Write([]byte(rel)) + hasher.Write([]byte{0}) + hasher.Write(content) + hasher.Write([]byte{0}) + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +// writeCompressedArtifact writes a cached artifact to +// outputDir/.br.b64, matching compressWorkflow's naming. +func writeCompressedArtifact(outputDir, workflowName string, artifact []byte) (string, error) { + if mkErr := os.MkdirAll(outputDir, 0o755); mkErr != nil { + return "", errors.Wrap(mkErr, "failed to prepare workflow build dir") + } + outputFile := filepath.Join(outputDir, workflowName+".br.b64") + if writeErr := os.WriteFile(outputFile, artifact, 0o644); writeErr != nil { //nolint:gosec // G306: we want it to be readable by everyone, same as compressWorkflow + return "", errors.Wrap(writeErr, "failed to write cached workflow artifact") + } + return outputFile, nil +} + +// compileWorkflowToDirUncached compiles the workflow and returns the path to +// the compressed artifact in outputDir. +func compileWorkflowToDirUncached(ctx context.Context, workflowFilePath, workflowName, outputDir string) (string, error) { language, lErr := delectLanguage(workflowFilePath) if lErr != nil { return "", errors.Wrap(lErr, "failed to detect workflow language") @@ -162,7 +343,7 @@ func compressWorkflow(workflowWasmPath string) (string, error) { } if err := os.WriteFile(outputFile, outputData, 0644); err != nil { //nolint:gosec // G306: we want it to be readable by everyone - return "", errors.Wrap(err, "failed to write output file") + return "", errors.Wrap(err, "failed to write compressed output file") } outputFileAbsPath, outputFileAbsPathErr := filepath.Abs(outputFile) diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 86b87f7c8eb..6ad7ead9188 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -37,11 +37,11 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260916140808-f5d22b3cb3ee github.com/smartcontractkit/chainlink-protos/job-distributor v0.20.1-0.20260701185448-696c075849ea @@ -492,7 +492,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 381c70b04cc..8567ffa90d4 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1654,20 +1654,20 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index ff3f84d756a..c40d721d96b 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -76,7 +76,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.111 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 @@ -624,8 +624,8 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 // indirect - github.com/smartcontractkit/chainlink-data-streams v1.1.1 // indirect - github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 // indirect + github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 // indirect + github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260723212800-b2f21d31b1d2 // indirect diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index e44e29a7837..a23aed2d89a 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1833,8 +1833,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260916222901-720a003dab50/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0 h1:3f+Ijgr0vghawMUAitgFILQpt4KsoYHGv1Zb/YBeeQM= github.com/smartcontractkit/chainlink-ccv v0.13.1-0.20260918171034-c93b0d2ef3c0/go.mod h1:gX7FJzSnGQx4RAlsn2rRW8IQBlEFZYiXFXEEB5jnQgA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f h1:gasuIea3r0AIuEkXDQvIw+TclWt184xBRZGwed+fP6I= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260928161532-9b1062f5076f/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49 h1:8vq1hWW7NR22x6+hpT727FREMQL4LqZuTpgKBTHVUKo= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260929093916-bc9452cf0a49/go.mod h1:2AvwgNr3I6s0Ny1qmovy/stHQIihEv/9g74aNYzQYYo= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260915184316-2730f1867c92 h1:Z1OIGukDhJD+Gknr5G07Vtml24E7BwKaIQoWzV/SdqA= @@ -1843,12 +1843,12 @@ github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61 github.com/smartcontractkit/chainlink-confidential-compute v1.3.0/go.mod h1:fq9n85XoREIUxgdXIX3RyDIRCRyxZKZ5NCBbFDfdySo= github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d h1:CZ0Om7lANyhpFJcmZE8RwQpEP4PyvoNLhOJckjm+zao= github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d/go.mod h1:Q5q/ohoAF5N7GXZS6QxgPWSaSLFtS6/53QHJbBUkCmU= -github.com/smartcontractkit/chainlink-data-streams v1.1.1 h1:WcNIy7fy/HVnjHKOm21pLwD7aMa0H6IY8mB3aJHKU4c= -github.com/smartcontractkit/chainlink-data-streams v1.1.1/go.mod h1:3mVEtx2PceMRauxmK6SJqJA7JB6XHeqCJLs370nYXgI= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23 h1:8TlbEKhx80HomyPpZGVy+zGSC1E1nnCRTTOLqa3Nvxs= +github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20260929142841-14ba41fe8d23/go.mod h1:ExmEs17bPBrsjV8XUs+Om5vvT/twPHeYfmAMwEnrtQQ= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2 h1:UtkIAbqWYtbExgCBEhQCJMEZpSrcAgPnk8GvFOGXuxA= github.com/smartcontractkit/chainlink-deployments-framework v0.122.2/go.mod h1:1fA5FLbgZlpC3OvLD/Nr86rgb4ZRI6MK0QEVVOIP1+o= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744 h1:n+V23NCROMeWFHUyR78Z4dXkUnPV3K8RzOzJpnXuXV0= -github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260917132551-2a2b38e8b744/go.mod h1:dEk48o9w++AyLCv12fNKIJf4Ko6iywWWJAGrjfS0vR8= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250 h1:4AMkR0hNrrnP/26zQbbzSCxOiLeh4oE1vavbls2Y93Y= +github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260929161505-cc645fce4250/go.mod h1:r8xuVq198XVbQSIfq5N5+W+ubnTmsjht/fn8xwAwYYw= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 h1:QJiXTG9CmaQAuMRn5JGi+Jhji7fSkehVnKpjc8oNJJY= github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501/go.mod h1:4cT1BeNF8DAn6In9zr3LayVCv1KzFeuxT7zcuNkfIb0= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260713161920-de075095648b h1:UdsGoTutNrzqQ23xA49U19lucVR98agzXRVPUPUf7eU= diff --git a/system-tests/tests/smoke/cre/vault_don_test.go b/system-tests/tests/smoke/cre/vault_don_test.go index 0b2908104e0..664e50662ed 100644 --- a/system-tests/tests/smoke/cre/vault_don_test.go +++ b/system-tests/tests/smoke/cre/vault_don_test.go @@ -14,37 +14,32 @@ import ( "testing" "time" + retry "github.com/avast/retry-go/v4" "github.com/ethereum/go-ethereum/common" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" - retry "github.com/avast/retry-go/v4" - vault_helpers "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" + workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common" workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events" + "github.com/smartcontractkit/chainlink-testing-framework/framework" + "github.com/smartcontractkit/chainlink-testing-framework/seth" keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset" crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm" - t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" - vaultcap "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes" - "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils" - - workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" - envconfig "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/vault" - ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" - - "github.com/smartcontractkit/chainlink-testing-framework/framework" - "github.com/smartcontractkit/chainlink-testing-framework/seth" - creworkflow "github.com/smartcontractkit/chainlink/system-tests/lib/cre/workflow" vaultsecret_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/vaultsecret/config" + t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" + ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" + vaultcap "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils" ) // ExecuteVaultAllowListBasedTests covers vault gateway + workflows with allow-listed JSON-RPC auth @@ -62,6 +57,10 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture gwURL := fixture.GatewayURL.String() vaultPublicKey := fixture.VaultPublicKey + // Shared by all subtests; checks are passed per HTTP trigger. Safe for the + // parallel identifier_validation subtest since its checks are negative-only. + verifier := deployVaultVerifierWorkflow(t, testEnv, fixture.TriggerAuth, "allowlist-verifier") + t.Run("allowlist_delete_batch_at_limit", func(t *testing.T) { sc := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient owner := sc.MustGetRootKeyAddress().Hex() @@ -189,7 +188,7 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture executeVaultAllowListSecretsCreateTest(t, createEnc, secretID, owner, owner, gwURL, namespaces, sc, wfReg) t.Run("binary_encoded_shares", func(t *testing.T) { - executeVaultBinaryEncodedSharesSmokeTest(t, testEnv, secretID, "main", createValue, ulCh, bmCh) + executeVaultBinaryEncodedSharesSmokeTest(t, verifier, secretID, "main", createValue, ulCh, bmCh) }) executeVaultSecretsUpdateTest(t, updateEnc, secretID, owner, owner, gwURL, namespaces, sc, wfReg) executeVaultSecretsListTest(t, secretID, owner, owner, gwURL, "main", sc, wfReg) @@ -202,13 +201,15 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture {Name: "allowlist-main-not-found", SecretKey: secretID, SecretNamespace: "main", ExpectNotFound: true}, {Name: "allowlist-alt-updated", SecretKey: secretID, SecretNamespace: "alt", ExpectedValue: updateValue}, } - workflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "allowlist-lifecycle", []vaultWorkflowPhase{ - {Name: "allowlist-updated", Checks: updatedChecks}, - {Name: "allowlist-final-verify", Checks: finalChecks}, - }) - waitForVaultWorkflowPhase(t, workflowID, "allowlist-updated", ulCh, bmCh) + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "allowlist-updated", + Checks: updatedChecks, + }, ulCh, bmCh) executeVaultSecretsDeleteTest(t, secretID, owner, owner, gwURL, []string{"main"}, sc, wfReg) - waitForVaultWorkflowPhase(t, workflowID, "allowlist-final-verify", ulCh, bmCh) + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "allowlist-final-verify", + Checks: finalChecks, + }, ulCh, bmCh) executeVaultSecretsDeleteTest(t, secretID, owner, owner, gwURL, []string{"alt"}, sc, wfReg) }) @@ -235,15 +236,15 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture t_helpers.ShutdownChipSinkWithDrain(ctx, sink, ulCh, bmCh) }) executeVaultSecretsIdentifierValidationTest(t, enc, owner, gwURL, sc, wfReg) - executeVaultSecretsGetInvalidIdentifierViaWorkflowTest(t, subEnv, "vget1", ulCh, bmCh) - executeVaultSecretsGetBatchTooBigViaWorkflowTest(t, subEnv, "vget2", ulCh, bmCh) + executeVaultSecretsGetInvalidIdentifierViaWorkflowTest(t, verifier, ulCh, bmCh) + executeVaultSecretsGetBatchTooBigViaWorkflowTest(t, verifier, ulCh, bmCh) }) t.Run("pending_queue_blob_batching_many_concurrent_creates", func(t *testing.T) { ExecuteVaultBlobBatchingSmokeTest(t, fixture, testEnv) }) t.Run("include_invalid_pending_items_liveness", func(t *testing.T) { - ExecuteVaultIncludeInvalidLivenessSmokeTest(t, fixture, testEnv) + ExecuteVaultIncludeInvalidLivenessSmokeTest(t, fixture, testEnv, verifier) }) } @@ -289,6 +290,9 @@ func ExecuteVaultMixedAuthTest(t *testing.T, fixture *vaultScenarioFixture, test jwtAuth := newJWTVaultRequestAuth(issuer, orgID, derivedJWTWorkflowOwner, vaultParsedPublicKey, false) workflowOwnerAddress := common.HexToAddress(workflowOwner) + // Shared by all mixed-auth phase checks below. + verifier := deployVaultVerifierWorkflow(t, testEnv, fixture.TriggerAuth, "mixed-auth-verifier") + t.Run("jwt_crud_with_workflow_owner", func(t *testing.T) { secretID := uniqueVaultSecretID("jwt") createValue := "secret-jwt-workflow-owner" @@ -407,39 +411,34 @@ func ExecuteVaultMixedAuthTest(t *testing.T, fixture *vaultScenarioFixture, test executeVaultSecretsCreateWithAuth(t, allowlistAuth, allowlistCreateEnc, allowlistSecretID, workflowOwner, gwURL, []string{"main"}) executeVaultSecretsCreateWithAuth(t, jwtAuth, jwtCreateEnc, jwtSecretID, derivedJWTWorkflowOwner, gwURL, []string{"main"}) // Workflow phases validate only allow-listed reads (workflow runs as EOA). JWT-backed keys are exercised via Gateway SecretsList. - workflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "mixed-lifecycle", []vaultWorkflowPhase{ - { - Name: "mixed-created", - Checks: []vaultWorkflowCheck{ - {Name: "mixed-allowlist-create-get-main", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectedValue: allowlistCreateValue}, - }, - }, - { - Name: "mixed-updated", - Checks: []vaultWorkflowCheck{ - {Name: "mixed-allowlist-own-update-main", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectedValue: allowlistUpdateValue}, - }, + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "mixed-created", + Checks: []vaultWorkflowCheck{ + {Name: "mixed-allowlist-create-get-main", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectedValue: allowlistCreateValue}, }, - { - Name: "mixed-deleted", - Checks: []vaultWorkflowCheck{ - {Name: "mixed-allowlist-delete-not-found", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectNotFound: true}, - }, - }, - }) - waitForVaultWorkflowPhase(t, workflowID, "mixed-created", ulCh, bmCh) + }, ulCh, bmCh) executeVaultSecretsListWithAuth(t, jwtAuth, []string{jwtSecretID}, derivedJWTWorkflowOwner, gwURL, "main") executeVaultSecretsUpdateWithAuth(t, allowlistAuth, allowlistUpdateEnc, allowlistSecretID, workflowOwner, gwURL, []string{"main"}) executeVaultSecretsUpdateWithAuth(t, jwtAuth, jwtUpdateEnc, jwtSecretID, derivedJWTWorkflowOwner, gwURL, []string{"main"}) - waitForVaultWorkflowPhase(t, workflowID, "mixed-updated", ulCh, bmCh) + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "mixed-updated", + Checks: []vaultWorkflowCheck{ + {Name: "mixed-allowlist-own-update-main", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectedValue: allowlistUpdateValue}, + }, + }, ulCh, bmCh) executeVaultSecretsListWithAuth(t, allowlistAuth, []string{allowlistSecretID}, workflowOwner, gwURL, "main") executeVaultSecretsListWithAuth(t, jwtAuth, []string{jwtSecretID}, derivedJWTWorkflowOwner, gwURL, "main") executeVaultSecretsDeleteWithAuth(t, allowlistAuth, allowlistSecretID, workflowOwner, gwURL, []string{"main"}) executeVaultSecretsDeleteWithAuth(t, jwtAuth, jwtSecretID, derivedJWTWorkflowOwner, gwURL, []string{"main"}) - waitForVaultWorkflowPhase(t, workflowID, "mixed-deleted", ulCh, bmCh) + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "mixed-deleted", + Checks: []vaultWorkflowCheck{ + {Name: "mixed-allowlist-delete-not-found", SecretKey: allowlistSecretID, SecretNamespace: "main", ExpectNotFound: true}, + }, + }, ulCh, bmCh) executeVaultJWTSecretsListAbsentFromNamespace(t, issuer, vaultParsedPublicKey, jwtSecretID, orgID, derivedJWTWorkflowOwner, gwURL, "main") }) @@ -453,23 +452,18 @@ func ExecuteVaultMixedAuthTest(t *testing.T, fixture *vaultScenarioFixture, test require.NoError(t, err) executeVaultSecretsCreateWithAuth(t, allowlistAuth, createEncAllow, sharedKey, workflowOwner, gwURL, []string{"main"}) - // Vault workflow returns after the FIRST phase whose checks succeed (see vaultsecret/main.go). - // Two phases cannot share the same success predicate; otherwise the later phase never emits - // "Vault secret workflow phase completed: ..." and waits time out (4m). - crossWorkflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "mixed-cross-isolation", []vaultWorkflowPhase{ - { - Name: "cross-isolation-stable", - Checks: []vaultWorkflowCheck{ - {Name: "cross-share-key-allowlist-value", SecretKey: sharedKey, SecretNamespace: "main", ExpectedValue: allowlistValue}, - }, + crossPhase := vaultWorkflowPhase{ + Name: "cross-isolation-stable", + Checks: []vaultWorkflowCheck{ + {Name: "cross-share-key-allowlist-value", SecretKey: sharedKey, SecretNamespace: "main", ExpectedValue: allowlistValue}, }, - }) - waitForVaultWorkflowPhase(t, crossWorkflowID, "cross-isolation-stable", ulCh, bmCh) + } + triggerAndAwaitVaultWorkflowPhase(t, verifier, crossPhase, ulCh, bmCh) // JWT cannot overwrite the allow-listed row: identifiers + labels are partitioned by owner address. tryJWTSignedVaultSecretsUpdate(t, jwtAuth, jwtAuth.requestOwner, jwtCrossEnc, sharedKey, gwURL, []string{"main"}) - // Confirm on a subsequent cron tick that secrets still satisfy the same invariant. - waitForVaultWorkflowPhase(t, crossWorkflowID, "cross-isolation-stable", ulCh, bmCh) + // Confirm on a subsequent trigger that secrets still satisfy the same invariant. + triggerAndAwaitVaultWorkflowPhase(t, verifier, crossPhase, ulCh, bmCh) executeVaultSecretsDeleteWithAuth(t, allowlistAuth, sharedKey, workflowOwner, gwURL, []string{"main"}) // `sharedKey` only ever existed under the allowlisted workflow owner; JWT-targeted deletes use the @@ -499,7 +493,7 @@ func ExecuteVaultMixedAuthTest(t *testing.T, fixture *vaultScenarioFixture, test // ExecuteVaultIncludeInvalidLivenessSmokeTest verifies that an erroring workflow GetSecrets for a // deleted secret does not stall concurrent valid gateway creates while include-invalid is enabled. -func ExecuteVaultIncludeInvalidLivenessSmokeTest(t *testing.T, fixture *vaultScenarioFixture, testEnv *ttypes.TestEnvironment) { +func ExecuteVaultIncludeInvalidLivenessSmokeTest(t *testing.T, fixture *vaultScenarioFixture, testEnv *ttypes.TestEnvironment, verifier *vaultVerifierHandle) { t.Helper() testLogger := framework.L @@ -533,15 +527,6 @@ func ExecuteVaultIncludeInvalidLivenessSmokeTest(t *testing.T, fixture *vaultSce t_helpers.ShutdownChipSinkWithDrain(ctx, sink, ulCh, bmCh) }) - workflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "include-invalid-liveness", []vaultWorkflowPhase{ - { - Name: "deleted-secret-not-found", - Checks: []vaultWorkflowCheck{ - {Name: "get-deleted-not-found", SecretKey: deletedSecretID, SecretNamespace: "main", ExpectNotFound: true}, - }, - }, - }) - liveCreateRequestID := uuid.New().String() liveCreateRequest := newVaultJSONRequest(t, liveCreateRequestID, vaulttypes.MethodSecretsCreate, &vault_helpers.CreateSecretsRequest{ RequestId: liveCreateRequestID, @@ -556,11 +541,30 @@ func ExecuteVaultIncludeInvalidLivenessSmokeTest(t *testing.T, fixture *vaultSce createErrCh <- tryValidateVaultSecretsCreateResponse(gwURL, liveCreateRequest, liveCreateRequestID, liveSecretID, []string{owner}, namespaces) }() - waitForVaultWorkflowPhase(t, workflowID, "deleted-secret-not-found", ulCh, bmCh) + waitForDeletedSecretNotFoundPhase(t, verifier, deletedSecretID, ulCh, bmCh) <-createDone require.NoError(t, <-createErrCh) } +// waitForDeletedSecretNotFoundPhase asserts the verifier workflow sees +// deletedSecretID as not found. +func waitForDeletedSecretNotFoundPhase( + t *testing.T, + verifier *vaultVerifierHandle, + deletedSecretID string, + ulCh chan *workflowevents.UserLogs, + bmCh chan *commonevents.BaseMessage, +) { + t.Helper() + + triggerAndAwaitVaultWorkflowPhase(t, verifier, vaultWorkflowPhase{ + Name: "deleted-secret-not-found", + Checks: []vaultWorkflowCheck{ + {Name: "get-deleted-not-found", SecretKey: deletedSecretID, SecretNamespace: "main", ExpectNotFound: true}, + }, + }, ulCh, bmCh) +} + // local pending queue and exercises batched pending-queue blobs (beyond the legacy per-blob single-item cap). func ExecuteVaultBlobBatchingSmokeTest(t *testing.T, fixture *vaultScenarioFixture, testEnv *ttypes.TestEnvironment) { t.Helper() @@ -989,28 +993,20 @@ func TestMustMintVaultJWTForRequest_UsesRawRequestDigest(t *testing.T) { } func executeVaultSecretsGetInvalidIdentifierViaWorkflowTest( - t *testing.T, testEnv *ttypes.TestEnvironment, - workflowBaseName string, + t *testing.T, verifier *vaultVerifierHandle, userLogsCh chan *workflowevents.UserLogs, baseMessageCh chan *commonevents.BaseMessage, ) { testLogger := framework.L testLogger.Info().Msg("Verifying get secret is rejected for invalid identifier via workflow...") - const workflowFileLocation = "./vaultsecret/main.go" - - workflowName := t_helpers.UniqueWorkflowName(testEnv, workflowBaseName) - workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &vaultsecret_config.Config{ + // Success is logged only after both the invalid key and invalid namespace are rejected. + triggerAndAwaitVaultVerifierLog(t, verifier, vaultsecret_config.TriggerInput{ + ExpectInvalidIdentifier: true, SecretKey: "invalid-key-with-hyphens", // hyphen not in [a-zA-Z0-9_]; tests invalid key SecretNamespace: "main", SecretKey2: "validkey", SecretNamespace2: "invalid-namespace-with-hyphens", // hyphen not in [a-zA-Z0-9_]; tests invalid namespace - ExpectInvalidIdentifier: true, - }, workflowFileLocation) - - // Both invalid-key and invalid-namespace checks run in the same cron trigger; a single - // success log is emitted only after both GetSecret calls are correctly rejected. - t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, - "Vault get correctly rejected invalid identifier", 4*time.Minute, t_helpers.WithUserLogWorkflowID(workflowID)) + }, "Vault get correctly rejected invalid identifier", userLogsCh, baseMessageCh) testLogger.Info().Msg("Vault get invalid identifier via workflow test completed") } @@ -1018,15 +1014,12 @@ func executeVaultSecretsGetInvalidIdentifierViaWorkflowTest( // GetSecrets batch is rejected with the real user-facing cause, asserting the // message survives the workflow-DON -> vault-DON remote capability hop. func executeVaultSecretsGetBatchTooBigViaWorkflowTest( - t *testing.T, testEnv *ttypes.TestEnvironment, - workflowBaseName string, + t *testing.T, verifier *vaultVerifierHandle, userLogsCh chan *workflowevents.UserLogs, baseMessageCh chan *commonevents.BaseMessage, ) { testLogger := framework.L testLogger.Info().Msg("Verifying get secret is rejected for oversized batch via workflow...") - const workflowFileLocation = "./vaultsecret/main.go" - // One over the limit; keys are valid identifiers so the batch size check // fires, not identifier validation. keys := make([]string, vaulttypes.MaxBatchSize+1) @@ -1034,15 +1027,11 @@ func executeVaultSecretsGetBatchTooBigViaWorkflowTest( keys[i] = fmt.Sprintf("batchkey%d", i) } - workflowName := t_helpers.UniqueWorkflowName(testEnv, workflowBaseName) - workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, &vaultsecret_config.Config{ + triggerAndAwaitVaultVerifierLog(t, verifier, vaultsecret_config.TriggerInput{ + ExpectBatchTooBig: true, SecretNamespace: "main", BatchSecretKeys: keys, - ExpectBatchTooBig: true, - }, workflowFileLocation) - - t_helpers.WatchWorkflowLogs(t, testLogger, userLogsCh, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog, - "Vault get correctly rejected oversized batch", 4*time.Minute, t_helpers.WithUserLogWorkflowID(workflowID)) + }, "Vault get correctly rejected oversized batch", userLogsCh, baseMessageCh) testLogger.Info().Msg("Vault get oversized batch via workflow test completed") } diff --git a/system-tests/tests/smoke/cre/vault_don_test_helpers.go b/system-tests/tests/smoke/cre/vault_don_test_helpers.go index 3534f160941..9684f756860 100644 --- a/system-tests/tests/smoke/cre/vault_don_test_helpers.go +++ b/system-tests/tests/smoke/cre/vault_don_test_helpers.go @@ -3,6 +3,7 @@ package cre import ( "bytes" "context" + "crypto/ecdsa" "encoding/hex" "encoding/json" "errors" @@ -23,20 +24,21 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" - "github.com/smartcontractkit/tdh2/go/tdh2/tdh2easy" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" + "github.com/smartcontractkit/tdh2/go/tdh2/tdh2easy" vault_helpers "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault" capabilitiespb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway" capabilities_registry_v2 "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/capabilities_registry_wrapper_v2" workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2" + "github.com/smartcontractkit/chainlink-protos/cre/go/values" commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common" workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events" "github.com/smartcontractkit/chainlink-testing-framework/framework" @@ -48,12 +50,14 @@ import ( "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm" stvault "github.com/smartcontractkit/chainlink/system-tests/lib/cre/vault" creworkflow "github.com/smartcontractkit/chainlink/system-tests/lib/cre/workflow" + libcrypto "github.com/smartcontractkit/chainlink/system-tests/lib/crypto" vaultsecret_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/vaultsecret/config" t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" vaultjwt "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault" "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes" "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils" + "github.com/smartcontractkit/chainlink/v2/core/utils" ) const ( @@ -266,6 +270,23 @@ type vaultScenarioFixture struct { LinkingService *stvault.TestLinkingService GatewayURL *url.URL VaultPublicKey string + // TriggerAuth signs HTTP trigger requests to verifier workflows. + TriggerAuth *vaultTriggerAuth +} + +// vaultTriggerAuth is the HTTP trigger key pair: Address is the workflow's +// authorized key, PrivateKey signs the request JWT. +type vaultTriggerAuth struct { + Address common.Address + PrivateKey *ecdsa.PrivateKey +} + +func newVaultTriggerAuth() (*vaultTriggerAuth, error) { + addr, privateKey, err := libcrypto.GenerateNewKeyPair() + if err != nil { + return nil, err + } + return &vaultTriggerAuth{Address: addr, PrivateKey: privateKey}, nil } type vaultWorkflowCheck struct { @@ -324,6 +345,9 @@ func setupVaultScenarioFixture(t *testing.T, baseConfig *ttypes.TestConfig, useP linkingService, err := stvault.EnsureSharedTestLinkingServiceStarted() require.NoError(t, err) + triggerAuth, err := newVaultTriggerAuth() + require.NoError(t, err, "failed to generate vault verifier trigger signing key") + var testEnv *ttypes.TestEnvironment if usePerTestKeys { testEnv = t_helpers.SetupTestEnvironmentWithPerTestKeys(t, baseConfig) @@ -342,6 +366,7 @@ func setupVaultScenarioFixture(t *testing.T, baseConfig *ttypes.TestConfig, useP LinkingService: linkingService, GatewayURL: gatewayURL, VaultPublicKey: vaultPublicKey, + TriggerAuth: triggerAuth, } } @@ -1314,63 +1339,200 @@ func executeVaultJWTSecretsCreateUnauthorizedWithExtraClaimsTest( require.Contains(t, jsonResponse.Error.Error(), expectedAuthError) } -func startVaultSecretsWorkflowPhasesTest( - t *testing.T, testEnv *ttypes.TestEnvironment, - workflowBaseName string, - phases []vaultWorkflowPhase, -) string { +// vaultVerifierHandle is a deployed verifier workflow and what's needed to +// trigger it via the gateway. +type vaultVerifierHandle struct { + WorkflowName string + WorkflowID string + WorkflowOwner string + GatewayURL string + TriggerAuth *vaultTriggerAuth +} + +const ( + vaultVerifierTriggerRetryInterval = 2 * time.Second + vaultVerifierTriggerAttemptWindow = 30 * time.Second + vaultVerifierTriggerDeadline = 3 * time.Minute +) + +// deployVaultVerifierWorkflow deploys the vaultsecret workflow. Checks are +// passed per trigger, so one deployment serves every scenario. +func deployVaultVerifierWorkflow(t *testing.T, testEnv *ttypes.TestEnvironment, triggerAuth *vaultTriggerAuth, workflowBaseName string) *vaultVerifierHandle { t.Helper() testLogger := framework.L - testLogger.Info(). - Str("workflow_base_name", workflowBaseName). - Int("phase_count", len(phases)). - Msg("Starting vault workflow phase verification") - workflowName := t_helpers.UniqueWorkflowName(testEnv, workflowBaseName) - cfgPhases := make([]vaultsecret_config.Phase, 0, len(phases)) - for _, phase := range phases { - cfgChecks := make([]vaultsecret_config.Check, 0, len(phase.Checks)) - for _, check := range phase.Checks { - cfgChecks = append(cfgChecks, vaultsecret_config.Check{ - Name: check.Name, - SecretKey: check.SecretKey, - SecretNamespace: check.SecretNamespace, - ExpectedValue: check.ExpectedValue, - ExpectNotFound: check.ExpectNotFound, - }) - } - cfgPhases = append(cfgPhases, vaultsecret_config.Phase{ - Name: phase.Name, - Checks: cfgChecks, - }) + cfg := &vaultsecret_config.Config{AuthorizedKey: triggerAuth.Address.Hex()} + const workflowFileLocation = "./vaultsecret/main.go" + workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, cfg, workflowFileLocation) + + require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0]) + sc := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient + gatewayURL := mustVaultGatewayURL(t, testEnv) + return &vaultVerifierHandle{ + WorkflowName: workflowName, + WorkflowID: workflowID, + WorkflowOwner: strings.ToLower(sc.MustGetRootKeyAddress().Hex()), + GatewayURL: gatewayURL.String(), + TriggerAuth: triggerAuth, } +} - cfg := &vaultsecret_config.Config{Phases: cfgPhases} - const workflowFileLocation = "./vaultsecret/main.go" - return t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, workflowName, cfg, workflowFileLocation) +// sendVaultVerifierTriggerRequest sends one signed HTTP trigger request. It +// returns a retryable error while the workflow is still loading or if the +// gateway rejects the request. +func sendVaultVerifierTriggerRequest(t *testing.T, handle *vaultVerifierHandle, input vaultsecret_config.TriggerInput) error { + t.Helper() + + inputJSON, err := json.Marshal(input) + require.NoError(t, err, "failed to marshal vault verifier trigger input") + + triggerPayload := gateway_common.HTTPTriggerRequest{ + Workflow: gateway_common.WorkflowSelector{ + WorkflowOwner: handle.WorkflowOwner, + WorkflowName: handle.WorkflowName, + WorkflowTag: creworkflow.DefaultWorkflowTag, + WorkflowID: handle.WorkflowID, + }, + Input: json.RawMessage(inputJSON), + } + params, err := json.Marshal(triggerPayload) + require.NoError(t, err, "failed to marshal vault verifier trigger request") + rawParams := json.RawMessage(params) + + // Fresh ID per attempt: duplicate IDs get a cached response without re-executing. + req := jsonrpc.Request[json.RawMessage]{ + Version: jsonrpc.JsonRpcVersion, + ID: uuid.New().String(), + Method: gateway_common.MethodWorkflowExecute, + Params: &rawParams, + } + + token, err := utils.CreateRequestJWT(req) + require.NoError(t, err, "failed to create vault verifier trigger JWT") + tokenString, err := token.SignedString(handle.TriggerAuth.PrivateKey) + require.NoError(t, err, "failed to sign vault verifier trigger JWT") + req.Auth = tokenString + + requestBody, err := json.Marshal(req) + require.NoError(t, err, "failed to marshal vault verifier trigger request body") + + httpReq, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handle.GatewayURL, bytes.NewBuffer(requestBody)) + require.NoError(t, err, "failed to create vault verifier trigger HTTP request") + httpReq.Header.Set("Content-Type", "application/jsonrpc") + httpReq.Header.Set("Accept", "application/json") + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute vault verifier trigger request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read vault verifier trigger response body: %w", err) + } + framework.L.Info(). + Str("workflow", handle.WorkflowName). + Int("status", resp.StatusCode). + Msgf("Vault verifier trigger response: %s", string(body)) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("vault verifier trigger request rejected with status %d: %s", resp.StatusCode, string(body)) + } + + var parsed jsonrpc.Response[json.RawMessage] + if err := json.Unmarshal(body, &parsed); err != nil { + return fmt.Errorf("failed to unmarshal vault verifier trigger response: %w", err) + } + if parsed.Error != nil { + return fmt.Errorf("vault verifier trigger request returned error: %s", parsed.Error.Error()) + } + if parsed.ID != req.ID { + return fmt.Errorf("vault verifier trigger response ID mismatch: got %q want %q", parsed.ID, req.ID) + } + return nil } -func waitForVaultWorkflowPhase( +// triggerAndAwaitVaultVerifierLog triggers the verifier and waits for +// expectedUserLog, re-triggering until the workflow is loaded and checks pass. +func triggerAndAwaitVaultVerifierLog( t *testing.T, - workflowID, phaseName string, + handle *vaultVerifierHandle, + input vaultsecret_config.TriggerInput, + expectedUserLog string, userLogsCh chan *workflowevents.UserLogs, baseMessageCh chan *commonevents.BaseMessage, ) { t.Helper() testLogger := framework.L - t_helpers.WatchWorkflowLogs( - t, - testLogger, - userLogsCh, - baseMessageCh, - t_helpers.WorkflowEngineInitErrorLog, - "Vault secret workflow phase completed: "+phaseName, - 1*time.Minute, - t_helpers.WithUserLogWorkflowID(workflowID), - ) - testLogger.Info().Str("phase_name", phaseName).Msg("Vault secret workflow phase completed") + + ctx, cancelFn := context.WithTimeoutCause(t.Context(), vaultVerifierTriggerDeadline, errors.New("timed out waiting for vault verifier workflow log")) + defer cancelFn() + cancelCtx, cancelCauseFn := context.WithCancelCause(ctx) + defer cancelCauseFn(nil) + + // Fail fast on engine init errors, as WatchWorkflowLogs does. + go func() { + t_helpers.FailOnBaseMessage(cancelCtx, cancelCauseFn, t, testLogger, baseMessageCh, + t_helpers.WorkflowEngineInitErrorLog, t_helpers.WithBaseMessageWorkflowID(handle.WorkflowID)) + }() + + var lastErr error + for { + if err := sendVaultVerifierTriggerRequest(t, handle, input); err != nil { + lastErr = err + testLogger.Warn().Str("workflow", handle.WorkflowName).Err(err).Msg("Vault verifier trigger not accepted yet; will retry") + } else { + attemptCtx, attemptCancel := context.WithTimeoutCause(cancelCtx, vaultVerifierTriggerAttemptWindow, errors.New("vault verifier trigger attempt window expired")) + _, waitErr := t_helpers.WaitForUserLog(attemptCtx, testLogger, userLogsCh, expectedUserLog, t_helpers.WithUserLogWorkflowID(handle.WorkflowID)) + attemptCancel() + if waitErr == nil { + testLogger.Info().Str("workflow", handle.WorkflowName).Str("expected_log", expectedUserLog).Msg("Vault verifier workflow log observed") + return + } + lastErr = waitErr + testLogger.Warn().Str("workflow", handle.WorkflowName).Err(waitErr).Msg("Vault verifier phase not yet satisfied; will re-trigger") + } + + select { + case <-cancelCtx.Done(): + require.FailNow(t, "timed out waiting for vault verifier workflow log", + "workflow=%s expectedLog=%q lastErr=%v", handle.WorkflowName, expectedUserLog, lastErr) + return + case <-time.After(vaultVerifierTriggerRetryInterval): + } + } +} + +// triggerAndAwaitVaultWorkflowPhase runs a single phase and waits for its completion log. +func triggerAndAwaitVaultWorkflowPhase( + t *testing.T, + handle *vaultVerifierHandle, + phase vaultWorkflowPhase, + userLogsCh chan *workflowevents.UserLogs, + baseMessageCh chan *commonevents.BaseMessage, +) { + t.Helper() + + input := vaultsecret_config.TriggerInput{Phases: []vaultsecret_config.Phase{vaultWorkflowPhaseToConfig(phase)}} + triggerAndAwaitVaultVerifierLog(t, handle, input, "Vault secret workflow phase completed: "+phase.Name, userLogsCh, baseMessageCh) +} + +func vaultWorkflowPhaseToConfig(phase vaultWorkflowPhase) vaultsecret_config.Phase { + checks := make([]vaultsecret_config.Check, 0, len(phase.Checks)) + for _, check := range phase.Checks { + checks = append(checks, vaultsecret_config.Check{ + Name: check.Name, + SecretKey: check.SecretKey, + SecretNamespace: check.SecretNamespace, + ExpectedValue: check.ExpectedValue, + ExpectNotFound: check.ExpectNotFound, + }) + } + return vaultsecret_config.Phase{Name: phase.Name, Checks: checks} } func executeVaultSecretsUpdateTest(t *testing.T, encryptedSecret, secretID, requestOwner, expectedResponseOwner, gatewayURL string, namespaces []string, sethClient *seth.Client, wfRegistryContract *workflow_registry_v2_wrapper.WorkflowRegistry) { @@ -1394,7 +1556,7 @@ func executeVaultSecretsDeleteTest(t *testing.T, secretID, requestOwner, expecte // workflow triggers a capability get. func executeVaultBinaryEncodedSharesSmokeTest( t *testing.T, - testEnv *ttypes.TestEnvironment, + handle *vaultVerifierHandle, secretID, namespace, expectedPlaintext string, userLogsCh chan *workflowevents.UserLogs, baseMessageCh chan *commonevents.BaseMessage, @@ -1403,7 +1565,7 @@ func executeVaultBinaryEncodedSharesSmokeTest( framework.L.Info().Msg("Verifying workflow secret fetch and binary share encoding when optimizations are enabled...") - workflowID := startVaultSecretsWorkflowPhasesTest(t, testEnv, "binary-shares", []vaultWorkflowPhase{{ + phase := vaultWorkflowPhase{ Name: "binary-shares-fetch", Checks: []vaultWorkflowCheck{{ Name: "binary-shares-main", @@ -1411,8 +1573,8 @@ func executeVaultBinaryEncodedSharesSmokeTest( SecretNamespace: namespace, ExpectedValue: expectedPlaintext, }}, - }}) - waitForVaultWorkflowPhase(t, workflowID, "binary-shares-fetch", userLogsCh, baseMessageCh) + } + triggerAndAwaitVaultWorkflowPhase(t, handle, phase, userLogsCh, baseMessageCh) } // updateVaultCapabilityConfigInRegistry updates the on-chain capabilities registry @@ -1503,8 +1665,8 @@ func updateVaultCapabilityConfigInRegistry(t *testing.T, testEnv *ttypes.TestEnv _, err = deployerClient.Decode(capReg.UpdateDONByName(deployerClient.NewTXOpts(), don.Name, updateParams)) require.NoError(t, err, "UpdateDONByName tx failed") - testLogger.Info().Msg("Waiting for registry syncer to propagate the on-chain config change...") - time.Sleep(15 * time.Second) // registry syncer polls every 12s; one tick + margin + // No wait: nodes pick up the config on their next registry syncer tick (12s). + testLogger.Info().Msg("Updated vault capability config in capabilities registry") } func allowlistRequest(t *testing.T, owner string, request jsonrpc.Request[json.RawMessage], sethClient *seth.Client, wfRegistryContract *workflow_registry_v2_wrapper.WorkflowRegistry) { diff --git a/system-tests/tests/smoke/cre/vaultsecret/config/config.go b/system-tests/tests/smoke/cre/vaultsecret/config/config.go index f6f61d8b7a5..c8177b5b184 100644 --- a/system-tests/tests/smoke/cre/vaultsecret/config/config.go +++ b/system-tests/tests/smoke/cre/vaultsecret/config/config.go @@ -1,63 +1,33 @@ package config +// Check is a single secret assertion evaluated by the workflow. type Check struct { - Name string `yaml:"name,omitempty"` - SecretKey string `yaml:"secretKey"` - SecretNamespace string `yaml:"secretNamespace"` - ExpectedValue string `yaml:"expectedValue,omitempty"` - ExpectNotFound bool `yaml:"expectNotFound"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + SecretKey string `yaml:"secretKey" json:"secretKey"` + SecretNamespace string `yaml:"secretNamespace" json:"secretNamespace"` + ExpectedValue string `yaml:"expectedValue,omitempty" json:"expectedValue,omitempty"` + ExpectNotFound bool `yaml:"expectNotFound" json:"expectNotFound"` } type Phase struct { - Name string `yaml:"name"` - Checks []Check `yaml:"checks"` + Name string `yaml:"name" json:"name"` + Checks []Check `yaml:"checks" json:"checks"` } type Config struct { - Phases []Phase `yaml:"phases"` - Checks []Check `yaml:"checks"` - - // Legacy single-check fields kept for compatibility with any older callers. - SecretKey string `yaml:"secretKey,omitempty"` - SecretNamespace string `yaml:"secretNamespace,omitempty"` - SecretKey2 string `yaml:"secretKey2"` - SecretNamespace2 string `yaml:"secretNamespace2"` - ExpectedValue string `yaml:"expectedValue,omitempty"` - ExpectNotFound bool `yaml:"expectNotFound,omitempty"` - ExpectInvalidIdentifier bool `yaml:"expectInvalidIdentifier"` - ExpectBatchTooBig bool `yaml:"expectBatchTooBig,omitempty"` - BatchSecretKeys []string `yaml:"batchSecretKeys,omitempty"` + // AuthorizedKey is the EVM address allowed to sign HTTP trigger requests. + AuthorizedKey string `yaml:"authorizedKey" json:"authorizedKey"` } -func (c Config) EffectiveChecks() []Check { - if len(c.Checks) > 0 { - return c.Checks - } - - if c.SecretKey == "" && c.SecretNamespace == "" { - return nil - } - - return []Check{{ - SecretKey: c.SecretKey, - SecretNamespace: c.SecretNamespace, - ExpectedValue: c.ExpectedValue, - ExpectNotFound: c.ExpectNotFound, - }} -} - -func (c Config) EffectivePhases() []Phase { - if len(c.Phases) > 0 { - return c.Phases - } - - checks := c.EffectiveChecks() - if len(checks) == 0 { - return nil - } - - return []Phase{{ - Name: "default", - Checks: checks, - }} +type TriggerInput struct { + // Phases are evaluated in order; the first whose checks all pass logs completion. + Phases []Phase `json:"phases,omitempty"` + + ExpectInvalidIdentifier bool `json:"expectInvalidIdentifier,omitempty"` + SecretKey string `json:"secretKey,omitempty"` + SecretNamespace string `json:"secretNamespace,omitempty"` + SecretKey2 string `json:"secretKey2,omitempty"` + SecretNamespace2 string `json:"secretNamespace2,omitempty"` + ExpectBatchTooBig bool `json:"expectBatchTooBig,omitempty"` + BatchSecretKeys []string `json:"batchSecretKeys,omitempty"` } diff --git a/system-tests/tests/smoke/cre/vaultsecret/go.mod b/system-tests/tests/smoke/cre/vaultsecret/go.mod index 936986b55dd..c03e186aca2 100644 --- a/system-tests/tests/smoke/cre/vaultsecret/go.mod +++ b/system-tests/tests/smoke/cre/vaultsecret/go.mod @@ -4,7 +4,7 @@ go 1.26.5 require ( github.com/smartcontractkit/cre-sdk-go v1.16.0 - github.com/smartcontractkit/cre-sdk-go/capabilities/scheduler/cron v1.3.0 + github.com/smartcontractkit/cre-sdk-go/capabilities/networking/http v1.4.1-0.20260617164717-4ae5e15fa5ac gopkg.in/yaml.v3 v3.0.1 ) diff --git a/system-tests/tests/smoke/cre/vaultsecret/go.sum b/system-tests/tests/smoke/cre/vaultsecret/go.sum index 2d4e1afc506..42d865f6eed 100644 --- a/system-tests/tests/smoke/cre/vaultsecret/go.sum +++ b/system-tests/tests/smoke/cre/vaultsecret/go.sum @@ -28,8 +28,8 @@ github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260707203317-661b54 github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260707203317-661b54b51a33/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= github.com/smartcontractkit/cre-sdk-go v1.16.0 h1:xFdLz8nf4pEzMkS+4Qq43aMvfl4EcnSshhsssM5ToIg= github.com/smartcontractkit/cre-sdk-go v1.16.0/go.mod h1:hhPdldSAv6CYSeHZzGOqFKDPe5zA6fuiBQpC0Ue/HI0= -github.com/smartcontractkit/cre-sdk-go/capabilities/scheduler/cron v1.3.0 h1:qBZ4y6qlTOynSpU1QAi2Fgr3tUZQ332b6hit9EVZqkk= -github.com/smartcontractkit/cre-sdk-go/capabilities/scheduler/cron v1.3.0/go.mod h1:Rzhy75vD3FqQo/SV6lypnxIwjWac6IOWzI5BYj3tYMU= +github.com/smartcontractkit/cre-sdk-go/capabilities/networking/http v1.4.1-0.20260617164717-4ae5e15fa5ac h1:t9WmKs/fjV+FiQmfGOCZzbQWorHVCcEQ4DRXBmczxw0= +github.com/smartcontractkit/cre-sdk-go/capabilities/networking/http v1.4.1-0.20260617164717-4ae5e15fa5ac/go.mod h1:GrSp5EegpvJNfQkBCN1klmrMBwaVTRAlvt020Yx0JJ4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= diff --git a/system-tests/tests/smoke/cre/vaultsecret/main.go b/system-tests/tests/smoke/cre/vaultsecret/main.go index 2382e9dd4b4..8c86e61e823 100644 --- a/system-tests/tests/smoke/cre/vaultsecret/main.go +++ b/system-tests/tests/smoke/cre/vaultsecret/main.go @@ -3,13 +3,14 @@ package main import ( + "encoding/json" "fmt" "log/slog" "strings" "gopkg.in/yaml.v3" - "github.com/smartcontractkit/cre-sdk-go/capabilities/scheduler/cron" + httptrigger "github.com/smartcontractkit/cre-sdk-go/capabilities/networking/http" "github.com/smartcontractkit/cre-sdk-go/cre" "github.com/smartcontractkit/cre-sdk-go/cre/wasm" @@ -26,25 +27,39 @@ func main() { }).Run(RunVaultSecretWorkflow) } +// RunVaultSecretWorkflow is an HTTP-triggered verifier; checks arrive per +// invocation in config.TriggerInput. func RunVaultSecretWorkflow(cfg config.Config, _ *slog.Logger, _ cre.SecretsProvider) (cre.Workflow[config.Config], error) { return cre.Workflow[config.Config]{ cre.Handler( - // cron-trigger@1.0.0 rejects schedules faster than 30s ("maximum fastest cron schedule is 30s"). - cron.Trigger(&cron.Config{Schedule: "*/30 * * * * *"}), + httptrigger.Trigger(&httptrigger.Config{ + AuthorizedKeys: []*httptrigger.AuthorizedKey{ + { + Type: httptrigger.KeyType_KEY_TYPE_ECDSA_EVM, + PublicKey: cfg.AuthorizedKey, + }, + }, + }), onTrigger, ), }, nil } -func onTrigger(cfg config.Config, runtime cre.Runtime, _ *cron.Payload) (string, error) { - if cfg.ExpectInvalidIdentifier { - return evaluateInvalidIdentifiers(cfg, runtime) +func onTrigger(cfg config.Config, runtime cre.Runtime, trigger *httptrigger.Payload) (string, error) { + var input config.TriggerInput + if len(trigger.Input) > 0 { + if err := json.Unmarshal(trigger.Input, &input); err != nil { + return "", fmt.Errorf("failed to unmarshal trigger input: %w", err) + } + } + if input.ExpectInvalidIdentifier { + return evaluateInvalidIdentifiers(input, runtime) } - if cfg.ExpectBatchTooBig { - return evaluateBatchTooBig(cfg, runtime) + if input.ExpectBatchTooBig { + return evaluateBatchTooBig(input, runtime) } - phases := cfg.EffectivePhases() + phases := input.Phases if len(phases) == 0 { return "", fmt.Errorf("no vault workflow phases configured") } @@ -73,25 +88,25 @@ func onTrigger(cfg config.Config, runtime cre.Runtime, _ *cron.Payload) (string, return "", fmt.Errorf("no vault workflow phase matched current state: %w", lastErr) } -func evaluateInvalidIdentifiers(cfg config.Config, runtime cre.Runtime) (string, error) { +func evaluateInvalidIdentifiers(input config.TriggerInput, runtime cre.Runtime) (string, error) { _, err := runtime.GetSecret(&cre.SecretRequest{ - Namespace: cfg.SecretNamespace, - Id: cfg.SecretKey, + Namespace: input.SecretNamespace, + Id: input.SecretKey, }).Await() if err == nil { - runtime.Logger().Error("Expected identifier validation to fail but GetSecret succeeded", "secretKey", cfg.SecretKey) - return "", fmt.Errorf("expected identifier validation failure for key=%s, but secret was retrieved", cfg.SecretKey) + runtime.Logger().Error("Expected identifier validation to fail but GetSecret succeeded", "secretKey", input.SecretKey) + return "", fmt.Errorf("expected identifier validation failure for key=%s, but secret was retrieved", input.SecretKey) } - runtime.Logger().Info("Vault get correctly rejected invalid identifier", "secretKey", cfg.SecretKey, "error", err) + runtime.Logger().Info("Vault get correctly rejected invalid identifier", "secretKey", input.SecretKey, "error", err) - if cfg.SecretKey2 != "" || cfg.SecretNamespace2 != "" { - key2 := cfg.SecretKey2 + if input.SecretKey2 != "" || input.SecretNamespace2 != "" { + key2 := input.SecretKey2 if key2 == "" { - key2 = cfg.SecretKey + key2 = input.SecretKey } - ns2 := cfg.SecretNamespace2 + ns2 := input.SecretNamespace2 if ns2 == "" { - ns2 = cfg.SecretNamespace + ns2 = input.SecretNamespace } _, err2 := runtime.GetSecret(&cre.SecretRequest{ Namespace: ns2, @@ -105,21 +120,21 @@ func evaluateInvalidIdentifiers(cfg config.Config, runtime cre.Runtime) (string, runtime.Logger().Info("Vault get correctly rejected invalid identifier", "secretKey", key2, "error", err2) } - return fmt.Sprintf("Invalid identifier correctly rejected: key=%s", cfg.SecretKey), nil + return fmt.Sprintf("Invalid identifier correctly rejected: key=%s", input.SecretKey), nil } // evaluateBatchTooBig submits a GetSecrets batch larger than the vault request // batch size limit and verifies the rejection carries the real user-facing // cause across the workflow-DON -> vault-DON remote capability hop. -func evaluateBatchTooBig(cfg config.Config, runtime cre.Runtime) (string, error) { - if len(cfg.BatchSecretKeys) == 0 { +func evaluateBatchTooBig(input config.TriggerInput, runtime cre.Runtime) (string, error) { + if len(input.BatchSecretKeys) == 0 { return "", fmt.Errorf("expectBatchTooBig requires batchSecretKeys to be set") } - reqs := make([]*cre.SecretRequest, len(cfg.BatchSecretKeys)) - for i, key := range cfg.BatchSecretKeys { + reqs := make([]*cre.SecretRequest, len(input.BatchSecretKeys)) + for i, key := range input.BatchSecretKeys { reqs[i] = &cre.SecretRequest{ - Namespace: cfg.SecretNamespace, + Namespace: input.SecretNamespace, Id: key, } }