From 8bf29038287c8f85c1999543ceefc1da2e8b9e8f Mon Sep 17 00:00:00 2001 From: Denys Kashkovskyi Date: Tue, 8 Sep 2026 00:35:25 +0200 Subject: [PATCH] Verify contributed source facts before graph assembly --- src/code_graph/checkpoint/commands.ts | 3 + src/code_graph/indexer_build.ts | 49 ++-- src/code_graph/indexer_materialization.ts | 7 + src/code_graph/indexer_service.ts | 120 +++++--- src/code_graph/indexer_types.ts | 20 +- src/code_graph/sharing/control_server.ts | 3 +- .../sharing/publication_evidence.ts | 19 ++ src/code_graph/sharing/publisher_cycle.ts | 260 ++++++++++-------- src/code_graph/sharing/source_verification.ts | 148 ++++++++++ ...graph.sharing-publication-evidence.test.ts | 193 ++++++++++--- ...graph.sharing-publication-evidence.test.ts | 8 + ...-graph.sharing-source-verification.test.ts | 151 ++++++++++ 12 files changed, 774 insertions(+), 207 deletions(-) create mode 100644 src/code_graph/sharing/source_verification.ts create mode 100644 test/unit/code-graph.sharing-source-verification.test.ts diff --git a/src/code_graph/checkpoint/commands.ts b/src/code_graph/checkpoint/commands.ts index c94b73789..fe695fc74 100644 --- a/src/code_graph/checkpoint/commands.ts +++ b/src/code_graph/checkpoint/commands.ts @@ -69,6 +69,8 @@ export interface CodeGraphCheckpointArtifactOptions { export interface CodeGraphCheckpointExportOptions { readonly cwd?: string; + /** @internal Bind publication to the physical snapshot whose assembly was verified. */ + readonly expectedSnapshotId?: string; readonly json?: boolean; readonly output: string; readonly quiet?: boolean; @@ -164,6 +166,7 @@ export const runCodeGraphCheckpointExport = Effect.fn('codeGraph.checkpoint.expo const snapshot = yield* store.readySnapshot(layout.databasePath, identity.worktreeId); if ( snapshot === undefined || + (options.expectedSnapshotId !== undefined && snapshot.id !== options.expectedSnapshotId) || snapshot.state !== 'ready' || snapshot.dirty || snapshot.commit !== identity.headCommit || diff --git a/src/code_graph/indexer_build.ts b/src/code_graph/indexer_build.ts index cbd809b88..49af88422 100644 --- a/src/code_graph/indexer_build.ts +++ b/src/code_graph/indexer_build.ts @@ -81,6 +81,7 @@ import { } from './indexer_snapshot_reuse.js'; import type { CodeGraphIndexOptions, + CodeGraphSourceVerification, CommittedBaseResult, DirectPersistentCapacityProtection, IncrementalOverlayAssessment, @@ -245,6 +246,7 @@ export const buildOwnedCleanSnapshot = Effect.fn('codeGraph.buildOwnedCleanSnaps readonly existing: CodeGraphSnapshot | undefined; readonly fallbackSnapshotId: string; readonly force: boolean; + readonly sourceVerification?: CodeGraphSourceVerification; readonly fs: FileSystem.FileSystem; readonly identity: RepositoryIdentity; readonly inventory: CodeGraphInventory; @@ -334,9 +336,10 @@ export const buildOwnedCleanSnapshot = Effect.fn('codeGraph.buildOwnedCleanSnaps cleanFallbackAssessment = reused.value; } } - const resumed = input.force - ? yield* input.store.resumableForcedBuild(input.layout.databasePath, input.logicalSnapshotId) - : undefined; + const resumed = + input.force && input.sourceVerification === undefined + ? yield* input.store.resumableForcedBuild(input.layout.databasePath, input.logicalSnapshotId) + : undefined; const building: CodeGraphSnapshot = resumed ?? { commit: input.identity.headCommit, dirty: false, @@ -365,6 +368,7 @@ export const buildOwnedCleanSnapshot = Effect.fn('codeGraph.buildOwnedCleanSnaps ensureVectors: input.ensureVectors, existing: input.existing, force: input.force, + sourceVerification: input.sourceVerification, fs: input.fs, identity: input.identity, incrementalAssessment: cleanFallbackAssessment, @@ -1005,6 +1009,7 @@ export const buildAndActivate = Effect.fn('codeGraph.buildAndActivate')(function readonly embedding: CodeGraphEmbeddingIndexShape; readonly ensureVectors: boolean; readonly force: boolean; + readonly sourceVerification?: CodeGraphSourceVerification; readonly fs: FileSystem.FileSystem; readonly identity: RepositoryIdentity; readonly inventory: CodeGraphInventory; @@ -1439,21 +1444,22 @@ export const buildAndActivate = Effect.fn('codeGraph.buildAndActivate')(function const loadingStartedAt = yield* Clock.currentTimeMillis; // Attribution may inspect peer facts in this deterministic source batch (for example, TypeScript barrels). // Reuse only a complete batch so a hit/miss partition cannot become a persisted derivation input. - const materializedShards = directPersistentMaterialization - ? yield* input.store.loadMaterializedFileShards( - input.layout.databasePath, - files, - input.building.extractorSet, - shardDerivationIdentity, - {currentGraphContentId, snapshotIds: donorSnapshotIds}, - ) - : { - bytes: 0, - bytesByPath: new Map(), - exactGenerationFiles: 0, - facts: new Map(), - materializedShardIdsByPath: new Map(), - }; + const materializedShards = + directPersistentMaterialization && input.sourceVerification === undefined + ? yield* input.store.loadMaterializedFileShards( + input.layout.databasePath, + files, + input.building.extractorSet, + shardDerivationIdentity, + {currentGraphContentId, snapshotIds: donorSnapshotIds}, + ) + : { + bytes: 0, + bytesByPath: new Map(), + exactGenerationFiles: 0, + facts: new Map(), + materializedShardIdsByPath: new Map(), + }; const exactGenerationShardFiles = materializedShards.exactGenerationFiles; const materializedShardBatchComplete = directPersistentMaterialization && @@ -1511,10 +1517,15 @@ export const buildAndActivate = Effect.fn('codeGraph.buildAndActivate')(function unit: 'files', }) ?? Effect.void; const attributionStartedAt = yield* Clock.currentTimeMillis; + // Original worker objects flow directly into this assembly; they never become raw-cache donors. + const materializationFacts = + input.sourceVerification === undefined + ? cached.facts + : yield* input.sourceVerification.materializeFacts({facts: cached.facts, files: fallbackFiles}); let flushShardCacheAfterAttribution = false; const attributedFallbackFacts = materializationSubphases.measure('attributionCompute', () => attributeFacts( - fallbackFiles.map(file => input.languagePacks.postprocessFile(file, cached.facts.get(file.path)!)), + fallbackFiles.map(file => input.languagePacks.postprocessFile(file, materializationFacts.get(file.path)!)), ), ); replayMetrics = addMaterializationReplayMetrics(replayMetrics, { diff --git a/src/code_graph/indexer_materialization.ts b/src/code_graph/indexer_materialization.ts index fae2a2ef2..3f58f6b40 100644 --- a/src/code_graph/indexer_materialization.ts +++ b/src/code_graph/indexer_materialization.ts @@ -201,6 +201,12 @@ function codeGraphFileProgressDimensions( export function cacheContentBatch(options: { readonly databasePath: string; readonly languagePacks: CodeGraphLanguagePackRegistryShape; + /** Required verification runs before persistence; unlike contribution enqueue, failures propagate. */ + readonly onSourceParserBatch?: (group: { + readonly cacheIdentity: string; + readonly facts: readonly BoundedCodeGraphFact[]; + readonly files: readonly CodeGraphInventoryFile[]; + }) => Effect.Effect; readonly onCachedParserBatch?: (group: { readonly cacheIdentity: string; readonly facts: readonly BoundedCodeGraphFact[]; @@ -326,6 +332,7 @@ export function cacheContentBatch(options: { currentScanningMetrics(), ); const startedAt = performance.now(); + yield* options.onSourceParserBatch?.(group) ?? Effect.void; yield* options.store.cacheFacts( options.databasePath, group.files, diff --git a/src/code_graph/indexer_service.ts b/src/code_graph/indexer_service.ts index c82ef6779..ec1a0b358 100644 --- a/src/code_graph/indexer_service.ts +++ b/src/code_graph/indexer_service.ts @@ -155,6 +155,17 @@ export class CodeGraphIndexer extends Context.Service enqueueSharedParserBatch(identity, options.threadnoteHome, group), + onSourceParserBatch: options.sourceVerification?.observeParserBatch, + onCachedParserBatch: options.sourceOnly + ? undefined + : group => enqueueSharedParserBatch(identity, options.threadnoteHome, group), onProgress: options.onProgress, parserPool, persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ @@ -492,9 +507,10 @@ export class CodeGraphIndexer extends Context.Service Schema.is(WorktreeChangedDuringIndex)(cause) && attempt === 0, + cause => + request.sourceVerification === undefined && Schema.is(WorktreeChangedDuringIndex)(cause) && attempt === 0, () => indexAttempt(request, anonymousTelemetry, attempt + 1, bypassCachedFacts), ), Effect.catchIf( - cause => Schema.is(CachedCodeGraphFactUnavailableDuringIndex)(cause) && !bypassCachedFacts, + cause => + request.sourceVerification === undefined && + Schema.is(CachedCodeGraphFactUnavailableDuringIndex)(cause) && + !bypassCachedFacts, () => indexAttempt(request, anonymousTelemetry, attempt, true), ), ); @@ -980,7 +1002,9 @@ export class CodeGraphIndexer extends Context.Service & {readonly commit: string}, + request: Omit & { + readonly commit: string; + }, anonymousTelemetry: CodeGraphBuildAnonymousTelemetryReporter, bypassCachedFacts = false, ): Effect.Effect<{readonly lease: CodeGraphCommitLease; readonly summary: CodeGraphIndexSummary}, unknown> => @@ -1035,7 +1059,7 @@ export class CodeGraphIndexer extends Context.Service enqueueSharedParserBatch(identity, options.threadnoteHome, group), + onCachedParserBatch: options.sourceOnly + ? undefined + : group => enqueueSharedParserBatch(identity, options.threadnoteHome, group), onProgress: options.onProgress, parserPool, persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ @@ -1202,10 +1229,11 @@ export class CodeGraphIndexer extends Context.Service & {readonly commit: string}, + request: Omit & { + readonly commit: string; + }, ) => Effect.flatMap( makeCodeGraphBuildAnonymousTelemetryReporter( diff --git a/src/code_graph/indexer_types.ts b/src/code_graph/indexer_types.ts index b04632343..d25bc6013 100644 --- a/src/code_graph/indexer_types.ts +++ b/src/code_graph/indexer_types.ts @@ -1,5 +1,6 @@ import {Crypto, Effect, Option, Path} from 'effect'; import type {SystemInfoShape} from '../effect/system.js'; +import type {BoundedCodeGraphFact} from './fact_budget.js'; import type {CodeGraphDirectPersistentCapacityBoundary} from './disk_capacity.js'; import type {CodeGraphIncrementalWork, CodeGraphIncrementalWorkObservation} from './incremental_work.js'; import type {CodeGraphInventoryOptions} from './inventory.js'; @@ -27,6 +28,10 @@ export interface CodeGraphIndexOptions extends CodeGraphInventoryOptions { /** Exact graph target supplied by a trusted local administration surface. */ readonly expectedIdentity?: RepositoryIdentityExpectation; readonly force?: boolean; + /** @internal Disable graph sharing import, hydration, enqueue and drain for source-only verification. */ + readonly sourceOnly?: boolean; + /** @internal Fail-closed hooks for a fresh, clean, forced source-only publication attempt. */ + readonly sourceVerification?: CodeGraphSourceVerification; /** Internal benchmark/correctness escape hatch; normal indexing keeps this enabled. */ readonly incrementalOverlay?: boolean; /** @internal Records read-back PRAGMA values for controlled benchmark evidence. */ @@ -43,6 +48,19 @@ export interface CodeGraphIndexOptions extends CodeGraphInventoryOptions { readonly threadnoteHome: string; } +export interface CodeGraphSourceVerification { + readonly observeParserBatch: (group: { + readonly cacheIdentity: string; + readonly facts: readonly BoundedCodeGraphFact[]; + readonly files: readonly CodeGraphInventoryFile[]; + }) => Effect.Effect; + /** Returned facts flow directly into postprocessing and attribution for this assembly batch. */ + readonly materializeFacts: (batch: { + readonly facts: ReadonlyMap; + readonly files: readonly CodeGraphInventoryFile[]; + }) => Effect.Effect, unknown>; +} + export interface DirectPersistentCapacityProtection { readonly availableDiskBytes: ( path: string, @@ -133,7 +151,7 @@ export interface CodeGraphCommitLease { export interface CodeGraphIndexerShape { readonly ensureCommit: ( - options: Omit & {readonly commit: string}, + options: Omit & {readonly commit: string}, ) => Effect.Effect; readonly index: (options: CodeGraphIndexOptions) => Effect.Effect; } diff --git a/src/code_graph/sharing/control_server.ts b/src/code_graph/sharing/control_server.ts index ed5505301..d4388084a 100644 --- a/src/code_graph/sharing/control_server.ts +++ b/src/code_graph/sharing/control_server.ts @@ -399,7 +399,8 @@ const commitCoordinatorDispatch = ( }), ); -function withCoordinatorStateLock( +/** @internal Serializes final publication checks with receipt/quarantine mutations. */ +export function withCoordinatorStateLock( options: Pick, effect: Effect.Effect, ) { diff --git a/src/code_graph/sharing/publication_evidence.ts b/src/code_graph/sharing/publication_evidence.ts index 408b6a675..a8c307976 100644 --- a/src/code_graph/sharing/publication_evidence.ts +++ b/src/code_graph/sharing/publication_evidence.ts @@ -1,9 +1,11 @@ import type {CodeGraphIndexSummary} from '../types.js'; import type {Sha256Digest} from './digest.js'; +import type {GraphShareSourceUseEvidence} from './source_verification.js'; const MAX_RESULT_DIGESTS = 128; export type GraphPublisherHydrationEvidence = + | {readonly status: 'skipped-source-verification'; readonly hydratedResults: 0} | {readonly status: 'completed'; readonly hydratedResults: number} | {readonly status: 'failed'; readonly hydratedResults: null}; @@ -22,6 +24,11 @@ export interface GraphPublisherContributionEvidence { readonly selectedResults: number; /** Receipt integrity and schema checks only; contributor identity and semantics are not authenticated. */ readonly verifiedResults: number; + /** Present only after all selected original payloads passed source verification and ready assembly. */ + readonly sourceUse?: Omit & { + readonly consumedResultManifestDigests: readonly Sha256Digest[]; + readonly resultDigestsTruncated: boolean; + }; } export function graphPublisherContributionEvidence(input: { @@ -30,6 +37,7 @@ export function graphPublisherContributionEvidence(input: { readonly snapshot: Pick; }; readonly selectedResults: number; + readonly sourceUse?: GraphShareSourceUseEvidence; readonly verifiedResultDigests: readonly Sha256Digest[]; }): GraphPublisherContributionEvidence { return { @@ -45,5 +53,16 @@ export function graphPublisherContributionEvidence(input: { resultManifestDigests: [...input.verifiedResultDigests].sort().slice(0, MAX_RESULT_DIGESTS), selectedResults: input.selectedResults, verifiedResults: input.verifiedResultDigests.length, + ...(input.sourceUse === undefined + ? {} + : { + sourceUse: { + ...input.sourceUse, + consumedResultManifestDigests: [...input.sourceUse.consumedResultManifestDigests] + .sort() + .slice(0, MAX_RESULT_DIGESTS), + resultDigestsTruncated: input.sourceUse.consumedResultManifestDigests.length > MAX_RESULT_DIGESTS, + }, + }), }; } diff --git a/src/code_graph/sharing/publisher_cycle.ts b/src/code_graph/sharing/publisher_cycle.ts index cb20c4b6f..4612c16cb 100644 --- a/src/code_graph/sharing/publisher_cycle.ts +++ b/src/code_graph/sharing/publisher_cycle.ts @@ -2,9 +2,7 @@ import {Clock, Crypto, Effect, FileSystem, Path, Ref} from 'effect'; import {runCodeGraphCheckpointExport} from '../checkpoint/commands.js'; import type {CodeGraphCheckpointHeaderV1, CodeGraphCheckpointRecordV1} from '../checkpoint/schema.js'; import {CodeGraphIndexer} from '../indexer.js'; -import {codeGraphDirectPersistentCapacityProtector} from '../indexer_materialization.js'; import {codeGraphLayout} from '../layout.js'; -import {CodeGraphMaintenanceCoordinator} from '../maintenance_coordinator.js'; import {resolveRepositoryIdentity} from '../repository.js'; import {CodeGraphStore} from '../store.js'; import {SystemInfo} from '../../effect/system.js'; @@ -30,7 +28,11 @@ import {decodeJsonBytes, readJsonFile, writePrivateJsonFile} from './atomic.js'; import {putCasFile, readVerifiedCasBlob} from './cas.js'; import {putGraphShareCheckpointLayers} from './checkpoint_cas.js'; import {putGraphShareOciDescriptor, putSignedGraphShareFrontierDocuments} from './descriptor.js'; -import {loadGraphShareCoordinatorState, updateGraphShareCoordinatorMachine} from './control_server.js'; +import { + loadGraphShareCoordinatorState, + updateGraphShareCoordinatorMachine, + withCoordinatorStateLock, +} from './control_server.js'; import type {GraphShareCoordinatorStateV1} from './control_protocol.js'; import {parseSha256Digest, type Sha256Digest} from './digest.js'; import {graphSharingFailure} from './errors.js'; @@ -52,11 +54,7 @@ import { graphShareCommitUnixSeconds, } from './git.js'; import {graphShareEnrollmentPath, graphSharingFrontierPointerPath, graphSharingLayout} from './layout.js'; -import { - hydratePublisherParseCache, - verifyGraphShareParseReceipt, - type VerifiedGraphShareParseReceipt, -} from './parse_cache.js'; +import {verifyGraphShareParseReceipt, type VerifiedGraphShareParseReceipt} from './parse_cache.js'; import { assertEnrollmentMatchesIdentity, parseGraphShareEnrollment, @@ -71,7 +69,7 @@ import { type GraphPublisherHydrationEvidence, } from './publication_evidence.js'; import {resolveGraphShareCasRoot} from './trust.js'; -import type {RepositoryIdentity} from '../types.js'; +import {makeGraphShareSourceVerification} from './source_verification.js'; export interface GraphPublisherCycleOptions { readonly cas?: string; @@ -173,6 +171,9 @@ export const advanceGraphPublisherFrontier = Effect.fn('codeGraph.sharing.advanc return currentPointer(current, pointer, machine.phase); } const selected = selectGraphShareResultsForFrozenMachine(coordinator.receipts, machine); + if (profilePointer.digest !== current.profileDigest) { + return yield* graphSharingFailure('Publisher enrollment profile differs from the current canonical frontier.'); + } const verified: VerifiedGraphShareParseReceipt[] = []; for (const announcement of selected.selected) { const receipt = yield* verifyGraphShareParseReceipt({ @@ -188,12 +189,12 @@ export const advanceGraphPublisherFrontier = Effect.fn('codeGraph.sharing.advanc } verified.push(receipt.value); } - const hydration: GraphPublisherHydrationEvidence = yield* hydratePublisherFacts(config, identity, verified).pipe( - Effect.match({ - onFailure: () => ({status: 'failed' as const, hydratedResults: null}), - onSuccess: result => ({status: 'completed' as const, hydratedResults: result.hydrated}), - }), - ); + const hydration: GraphPublisherHydrationEvidence = {status: 'skipped-source-verification', hydratedResults: 0}; + const verification = makeGraphShareSourceVerification({ + repositoryId: identity.repositoryId, + sourceCommit: identity.headCommit, + verified, + }); const published = yield* Effect.gen(function* () { const indexer = yield* CodeGraphIndexer; const store = yield* CodeGraphStore; @@ -201,6 +202,9 @@ export const advanceGraphPublisherFrontier = Effect.fn('codeGraph.sharing.advanc cwd, ensureVectors: false, force: true, + includeOverlay: false, + sourceOnly: true, + sourceVerification: verification.hooks, threadnoteHome: config.agentContextHome, }); const layout = codeGraphLayout(path, config.agentContextHome, identity.checkoutId, identity.worktreeId); @@ -215,17 +219,26 @@ export const advanceGraphPublisherFrontier = Effect.fn('codeGraph.sharing.advanc 'Checkpoint export requires the exact ready CLEAN root snapshot for the current repository HEAD.', ); } + if (ready.id !== indexed.snapshot.id) { + return yield* graphSharingFailure('The ready graph changed after source-verified assembly.'); + } + const sourceUse = yield* verification.complete(); machine = assembleGraphShareBatch(machine); yield* persistMachine(coordinatorOptions, machine, options.onMachine, options.stateRef); machine = verifyGraphShareBatch(machine); yield* persistMachine(coordinatorOptions, machine, options.onMachine, options.stateRef); - const exported = yield* exportSignedGeneration(config, options, current, identity.repositoryId, profile); + const exported = yield* exportSignedGeneration(config, options, current, identity.repositoryId, profile, { + snapshotId: ready.id, + sourceCommit: identity.headCommit, + verified, + }); return { ...exported, contributionEvidence: graphPublisherContributionEvidence({ hydration, index: indexed, selectedResults: selected.selected.length, + sourceUse, verifiedResultDigests: verified.map(item => item.announcement.resultManifestDigest), }), }; @@ -298,46 +311,17 @@ export const ensureGraphSharePublishedOciDescriptor = Effect.fn('codeGraph.shari }, ); -const hydratePublisherFacts = Effect.fn('codeGraph.sharing.hydratePublisherFacts')(function* ( - config: RuntimeConfig, - identity: RepositoryIdentity, - verified: readonly VerifiedGraphShareParseReceipt[], -) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const store = yield* CodeGraphStore; - const maintenance = yield* CodeGraphMaintenanceCoordinator; - const crypto = yield* Crypto.Crypto; - const system = yield* SystemInfo; - const layout = codeGraphLayout(path, config.agentContextHome, identity.checkoutId, identity.worktreeId); - return yield* hydratePublisherParseCache({ - databasePath: layout.databasePath, - persistentCapacityProtector: codeGraphDirectPersistentCapacityProtector({ - capacityProtection: { - availableDiskBytes: (target: string) => system.availableDiskBytes(target), - crypto, - maintenance, - path, - system, - temporaryDirectory: system.tempDirectory, - walAutoCheckpointPages: 1_000, - }, - fs, - identity, - layout, - threadnoteHome: config.agentContextHome, - }), - store, - verified, - }); -}); - const exportSignedGeneration = Effect.fn('codeGraph.sharing.exportSignedGeneration')(function* ( config: RuntimeConfig, options: GraphPublisherCycleOptions, current: GraphShareFrontierManifestV1, repositoryId: string, profile: GraphShareProfileV1, + expected: { + readonly snapshotId: string; + readonly sourceCommit: string; + readonly verified: readonly VerifiedGraphShareParseReceipt[]; + }, ) { const crypto = yield* Crypto.Crypto; const fs = yield* FileSystem.FileSystem; @@ -347,7 +331,12 @@ const exportSignedGeneration = Effect.fn('codeGraph.sharing.exportSignedGenerati const casRoot = yield* resolveGraphShareCasRoot(config.agentContextHome, options.cas); const key = yield* loadPublisherKey(config.agentContextHome); const spool = path.join(casRoot, 'spool', `${yield* crypto.randomUUIDv4}.cgcp`); - const exported = yield* runCodeGraphCheckpointExport(config, {cwd, output: spool, quiet: true}); + const exported = yield* runCodeGraphCheckpointExport(config, { + cwd, + expectedSnapshotId: expected.snapshotId, + output: spool, + quiet: true, + }); const exportedBytes = yield* fs.readFile(spool); const target = decodeGraphShareCheckpointBytes(exportedBytes); const previous = yield* loadPublishedTargetGraph(casRoot, current); @@ -390,66 +379,119 @@ const exportSignedGeneration = Effect.fn('codeGraph.sharing.exportSignedGenerati ? undefined : {metadataDigest: current.checkpoint.metadataDigest}; const publishedDelta = publication === 'delta' ? yield* putGraphShareDeltaArtifact(casRoot, deltaPack) : undefined; - const signed = yield* signGraphShareFrontier(key, { - branch: current.branch, - checkpoint: - publication === 'compact' - ? { - manifestDigest: checkpointDigest, - metadataDigest: checkpointLayers?.metadataDigest, - snapshotId: exported.snapshotId, - sourceCommit: exported.sourceCommit, - } - : current.checkpoint, - deltas: - publication === 'compact' || publishedDelta === undefined - ? [] - : [ - ...current.deltas, - { - baseSnapshotId: current.snapshotId, - manifestDigest: publishedDelta.digest, - metadataDigest: publishedDelta.layers.metadataDigest, - targetCommit: exported.sourceCommit, - targetSnapshotId: exported.snapshotId, - }, - ], - generation: current.generation + 1, - graphAbi: exported.graphAbi, - graphContentId: exported.graphContentId, - logicalGraphDigest: parseSha256Digest(exported.logicalDigest), - previousManifestDigest: graphShareFrontierDigest(current), - profileDigest: current.profileDigest, - publisherFence: current.publisherFence, + const coordinatorOptions = { + organization: profile.organization, repositoryId, - schemaVersion: 1, - snapshotId: exported.snapshotId, - sourceCommit: exported.sourceCommit, - }); - const metadataDigest = - publication === 'compact' - ? checkpointLayers?.metadataDigest - : (current.checkpoint.metadataDigest ?? publishedDelta?.layers.metadataDigest); - if (metadataDigest === undefined) { - return yield* graphSharingFailure('Frontier publication is missing checkpoint metadata.'); - } - const metadataBytes = yield* readVerifiedCasBlob(casRoot, metadataDigest); - const documents = yield* putSignedGraphShareFrontierDocuments(casRoot, signed, metadataBytes); - const layout = graphSharingLayout(path, config.agentContextHome, casRoot); - yield* writePrivateJsonFile(graphSharingFrontierPointerPath(path, layout.frontiersRoot, repositoryId), { - envelopeDigest: documents.envelopeDigest, - manifestDigest: documents.manifestDigest, - schemaVersion: 1, - }); - return { - checkpointDigest, - descriptorDigest: documents.descriptorDigest, - envelopeDigest: documents.envelopeDigest, - generation: current.generation + 1, - manifestDigest: documents.manifestDigest, - profileDigest: current.profileDigest, - sourceCommit: exported.sourceCommit, + threadnoteHome: config.agentContextHome, }; + // Serialize the last quarantine check and canonical pointer promotion with receipt acceptance. + return yield* withCoordinatorStateLock( + coordinatorOptions, + Effect.gen(function* () { + const verifyTarget = Effect.gen(function* () { + const latestIdentity = yield* resolveRepositoryIdentity(cwd); + const layout = graphSharingLayout(path, config.agentContextHome, casRoot); + const latestPointer = parseGraphShareFrontierPointer( + yield* readJsonFile(graphSharingFrontierPointerPath(path, layout.frontiersRoot, repositoryId)), + ); + const latestEnrollment = parseGraphShareEnrollment( + yield* readJsonFile(graphShareEnrollmentPath(path, latestIdentity.repoRoot)), + ); + if ( + latestIdentity.repositoryId !== repositoryId || + latestIdentity.headCommit !== expected.sourceCommit || + exported.sourceCommit !== expected.sourceCommit || + latestPointer.manifestDigest !== graphShareFrontierDigest(current) || + parseGraphShareProfilePointer(latestEnrollment.profile).digest !== current.profileDigest + ) { + return yield* graphSharingFailure('Publication source, profile, or predecessor changed during verification.'); + } + const state = + options.stateRef === undefined + ? yield* loadGraphShareCoordinatorState(coordinatorOptions) + : yield* Ref.get(options.stateRef); + const quarantine = new Set(state.receipts.quarantine.map(item => item.actionKey)); + if ( + expected.verified.some( + item => + quarantine.has(item.announcement.actionKey) || + !state.receipts.receipts.some( + receipt => + receipt.actionKey === item.announcement.actionKey && + receipt.resultManifestDigest === item.announcement.resultManifestDigest && + receipt.batchId === item.announcement.batchId, + ), + ) + ) { + return yield* graphSharingFailure( + 'A selected contribution changed or entered quarantine before publication.', + ); + } + }); + yield* verifyTarget; + const signed = yield* signGraphShareFrontier(key, { + branch: current.branch, + checkpoint: + publication === 'compact' + ? { + manifestDigest: checkpointDigest, + metadataDigest: checkpointLayers?.metadataDigest, + snapshotId: exported.snapshotId, + sourceCommit: exported.sourceCommit, + } + : current.checkpoint, + deltas: + publication === 'compact' || publishedDelta === undefined + ? [] + : [ + ...current.deltas, + { + baseSnapshotId: current.snapshotId, + manifestDigest: publishedDelta.digest, + metadataDigest: publishedDelta.layers.metadataDigest, + targetCommit: exported.sourceCommit, + targetSnapshotId: exported.snapshotId, + }, + ], + generation: current.generation + 1, + graphAbi: exported.graphAbi, + graphContentId: exported.graphContentId, + logicalGraphDigest: parseSha256Digest(exported.logicalDigest), + previousManifestDigest: graphShareFrontierDigest(current), + profileDigest: current.profileDigest, + publisherFence: current.publisherFence, + repositoryId, + schemaVersion: 1, + snapshotId: exported.snapshotId, + sourceCommit: exported.sourceCommit, + }); + const metadataDigest = + publication === 'compact' + ? checkpointLayers?.metadataDigest + : (current.checkpoint.metadataDigest ?? publishedDelta?.layers.metadataDigest); + if (metadataDigest === undefined) { + return yield* graphSharingFailure('Frontier publication is missing checkpoint metadata.'); + } + const metadataBytes = yield* readVerifiedCasBlob(casRoot, metadataDigest); + const documents = yield* putSignedGraphShareFrontierDocuments(casRoot, signed, metadataBytes); + const layout = graphSharingLayout(path, config.agentContextHome, casRoot); + yield* verifyTarget; + yield* writePrivateJsonFile(graphSharingFrontierPointerPath(path, layout.frontiersRoot, repositoryId), { + envelopeDigest: documents.envelopeDigest, + manifestDigest: documents.manifestDigest, + schemaVersion: 1, + }); + return { + checkpointDigest, + descriptorDigest: documents.descriptorDigest, + envelopeDigest: documents.envelopeDigest, + generation: current.generation + 1, + manifestDigest: documents.manifestDigest, + profileDigest: current.profileDigest, + sourceCommit: exported.sourceCommit, + }; + }), + ); }); const loadPublishedTargetGraph = Effect.fn('codeGraph.sharing.loadPublishedTarget')(function* ( diff --git a/src/code_graph/sharing/source_verification.ts b/src/code_graph/sharing/source_verification.ts new file mode 100644 index 000000000..48ed97d92 --- /dev/null +++ b/src/code_graph/sharing/source_verification.ts @@ -0,0 +1,148 @@ +import {Effect} from 'effect'; +import {canonicalJson} from '../checkpoint/canonical_json.js'; +import {serializeBoundedCodeGraphFact} from '../fact_budget.js'; +import type {CodeGraphSourceVerification} from '../indexer_types.js'; +import type {CodeGraphFileFacts, CodeGraphInventoryFile} from '../types.js'; +import {graphShareLanguageAndRole, graphShareParseActionKey} from './action.js'; +import type {Sha256Digest} from './digest.js'; +import {sha256Digest} from './digest.js'; +import {graphSharingFailure} from './errors.js'; +import type {VerifiedGraphShareParseReceipt} from './parse_cache.js'; + +export interface GraphShareSourceUseEvidence { + readonly consumedActions: number; + readonly consumedResultManifestDigests: readonly Sha256Digest[]; + readonly sourceVerifiedFiles: number; +} + +/** Attempt-local proof. Raw cache contents alone never establish source provenance. */ +export function makeGraphShareSourceVerification(input: { + readonly repositoryId: string; + readonly sourceCommit: string; + readonly verified: readonly VerifiedGraphShareParseReceipt[]; +}): { + readonly hooks: CodeGraphSourceVerification; + readonly complete: () => Effect.Effect; +} { + const selected = new Map(); + const fresh = new Map(); + const consumed = new Map(); + let initialized = false; + const initialize = () => { + if (initialized) return; + for (const item of input.verified) { + if (item.announcement.batchId !== input.sourceCommit || item.parsed.repositoryId !== input.repositoryId) { + throw graphSharingFailure('Contribution does not belong to the exact publication source target.'); + } + const previous = selected.get(item.parsed.normalizedPath); + if (previous && previous.announcement.resultManifestDigest !== item.announcement.resultManifestDigest) { + throw graphSharingFailure('Publication selected conflicting results for one source path.'); + } + selected.set(item.parsed.normalizedPath, item); + } + initialized = true; + }; + const digest = (facts: CodeGraphFileFacts) => sha256Digest(canonicalJson(facts)); + const context = (file: CodeGraphInventoryFile) => + canonicalJson({ + blobId: file.blobId, + contentHash: file.contentHash, + language: file.language, + mode: file.mode, + path: file.path, + source: file.source, + }); + const hooks: CodeGraphSourceVerification = { + observeParserBatch: group => + sourceVerificationAttempt(() => { + initialize(); + const factsByPath = new Map(group.facts.map(fact => [fact.facts.path, fact])); + for (const file of group.files) { + const local = factsByPath.get(file.path); + if (!local || file.source !== 'commit') { + throw graphSharingFailure('Source verification requires fresh committed parser facts.'); + } + const observed = {context: context(file), digest: digest(local.facts)}; + const previous = fresh.get(file.path); + if (previous && (previous.context !== observed.context || previous.digest !== observed.digest)) { + throw graphSharingFailure('Source facts changed within the publication attempt.'); + } + const item = selected.get(file.path); + if (item) { + const parsed = item.parsed; + const languageAndRole = graphShareLanguageAndRole(file.language, 'source'); + const actionKey = graphShareParseActionKey({ + contentHash: file.contentHash, + extractorSet: group.cacheIdentity, + languageAndRole, + normalizedPath: file.path, + repositoryId: input.repositoryId, + }); + if ( + parsed.actionKey !== actionKey || + item.announcement.actionKey !== actionKey || + parsed.gitBlobId !== file.blobId || + parsed.contentHash !== file.contentHash || + parsed.extractorSet !== group.cacheIdentity || + parsed.languageAndRole !== languageAndRole || + digest(parsed.facts) !== observed.digest + ) { + throw graphSharingFailure( + 'Contribution does not match independently parsed source facts and action context.', + ); + } + const bounded = serializeBoundedCodeGraphFact(parsed.facts); + if (digest(bounded.facts) !== observed.digest) { + throw graphSharingFailure('Contribution changes at the bounded fact representation boundary.'); + } + } + fresh.set(file.path, observed); + } + }), + materializeFacts: batch => + sourceVerificationAttempt(() => { + initialize(); + const output = new Map(batch.facts); + for (const file of batch.files) { + const local = batch.facts.get(file.path); + const observed = fresh.get(file.path); + if (!local || !observed || observed.context !== context(file) || digest(local) !== observed.digest) { + throw graphSharingFailure('Assembly input lacks matching fresh-source evidence from this attempt.'); + } + const item = selected.get(file.path); + if (!item) continue; + // Use the original payload's complete bounded value, never a rewritten or locally generated receipt. + const original = serializeBoundedCodeGraphFact(item.parsed.facts).facts; + if (digest(original) !== observed.digest) { + throw graphSharingFailure('Original contribution changed before assembly consumption.'); + } + output.set(file.path, original); + consumed.set(item.parsed.actionKey, item.announcement.resultManifestDigest); + } + return output; + }), + }; + return { + hooks, + // Call only after a successful ready assembly. A failed/interrupted attempt must discard this object. + complete: () => + sourceVerificationAttempt(() => { + initialize(); + if (selected.size !== consumed.size) { + throw graphSharingFailure('Publication did not consume every selected contribution.'); + } + return { + consumedActions: consumed.size, + consumedResultManifestDigests: [...consumed.values()].sort(), + sourceVerifiedFiles: fresh.size, + }; + }), + }; +} + +function sourceVerificationAttempt(body: () => A) { + return Effect.try({ + try: body, + catch: cause => graphSharingFailure('Contribution source verification failed.', cause), + }); +} diff --git a/test/integration/code-graph.sharing-publication-evidence.test.ts b/test/integration/code-graph.sharing-publication-evidence.test.ts index 76ad1179d..816e64041 100644 --- a/test/integration/code-graph.sharing-publication-evidence.test.ts +++ b/test/integration/code-graph.sharing-publication-evidence.test.ts @@ -1,19 +1,30 @@ import {describe, expect, it as effectIt} from '@effect/vitest'; -import {Deferred, Effect, Fiber, FileSystem, Path} from 'effect'; +import {Cause, Crypto, Deferred, Effect, Exit, Fiber, FileSystem, Path} from 'effect'; import {TestClock} from 'effect/testing'; import {provideTestLayer} from '../helpers/effect-layer.js'; import {ApplicationLayer} from '../../src/effect/runtime.js'; import {runCommandEffect} from '../../src/effect/command.js'; import {CodeGraphIndexer} from '../../src/code_graph/indexer.js'; import {CodeGraphStore} from '../../src/code_graph/store.js'; -import {CodeGraphStoreError} from '../../src/code_graph/types.js'; import {codeGraphLayout} from '../../src/code_graph/layout.js'; import {resolveRepositoryIdentity} from '../../src/code_graph/repository.js'; import {runCodeGraphCheckpointExport} from '../../src/code_graph/checkpoint/commands.js'; import {runGraphShareJoin} from '../../src/code_graph/sharing/client.js'; import {graphShareControlGetStatus} from '../../src/code_graph/sharing/control_client.js'; import {parseSha256Digest} from '../../src/code_graph/sharing/digest.js'; -import {verifyGraphShareParseReceipt} from '../../src/code_graph/sharing/parse_cache.js'; +import { + verifyGraphShareParseReceipt, + type VerifiedGraphShareParseReceipt, +} from '../../src/code_graph/sharing/parse_cache.js'; +import {graphSharingLayout, graphSharingFrontierPointerPath} from '../../src/code_graph/sharing/layout.js'; +import {putCasBytes} from '../../src/code_graph/sharing/cas.js'; +import {canonicalJson} from '../../src/code_graph/checkpoint/canonical_json.js'; +import {graphShareParseResultArtifact} from '../../src/code_graph/sharing/parse_result.js'; +import {readJsonFile, writePrivateJsonFile} from '../../src/code_graph/sharing/atomic.js'; +import {loadGraphShareCoordinatorState} from '../../src/code_graph/sharing/control_server.js'; +import {writeGraphShareCoordinatorUrl, writeGraphShareContributionMode} from '../../src/code_graph/sharing/trust.js'; +import {announceGraphShareResult} from '../../src/code_graph/sharing/receipts.js'; +import {sha256Digest} from '../../src/code_graph/sharing/digest.js'; import {advanceGraphPublisherFrontier} from '../../src/code_graph/sharing/publisher_cycle.js'; import { runGraphPublisherBootstrap, @@ -22,16 +33,22 @@ import { } from '../../src/code_graph/sharing/publisher.js'; describe('publisher contribution evidence with an independent clean control', () => { - for (const failHydration of [false, true]) { + for (const scenario of ['valid', 'forged', 'quarantine', 'abi-transition'] as const) { effectIt.effect( - failHydration - ? 'reports partial hydration failure while publishing a correct graph' - : 'distinguishes received facts from canonical publisher recomputation', + scenario === 'abi-transition' + ? 'compacts an independently verified graph when the active language-pack ABI changes' + : scenario === 'quarantine' + ? 'rejects a contribution quarantined after assembly and before promotion' + : scenario === 'forged' + ? 'rejects a validly hashed forged payload without promotion and verifies a clean retry' + : 'uses source-verified original contributions with an independent clean graph control', () => TestClock.withLive( Effect.gen(function* () { + const eligibleFiles = scenario === 'abi-transition' ? 4 : 3; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; const indexer = yield* CodeGraphIndexer; const store = yield* CodeGraphStore; const root = yield* fs.makeTempDirectoryScoped({prefix: 'threadnote-publication-evidence-'}); @@ -90,15 +107,21 @@ describe('publisher contribution evidence with an independent clean control', () path.join(contributor, 'src/b.ts'), "import {alpha} from './a.js'; export function beta(value: number) { return alpha(value) + 3; }\n", ); + if (scenario === 'abi-transition') + yield* fs.writeFileString( + path.join(contributor, 'src/new.py'), + 'def added_python(value):\n return value + 1\n', + ); yield* commit(contributor, 'target'); const target = yield* resolveRepositoryIdentity(contributor); yield* indexer.index({cwd: contributor, ensureVectors: false, threadnoteHome: workerHome}); const status = yield* graphShareControlGetStatus(url); const receipts = status.receipts.filter(receipt => receipt.batchId === target.headCommit); - expect(receipts).toHaveLength(3); + expect(receipts).toHaveLength(eligibleFiles); const identity = yield* resolveRepositoryIdentity(repo); expect(identity.headCommit).toBe(baseline.sourceCommit); const databasePath = codeGraphLayout(path, home, identity.checkoutId, identity.worktreeId).databasePath; + const verifiedReceipts: VerifiedGraphShareParseReceipt[] = []; for (const announcement of receipts) { const verified = yield* verifyGraphShareParseReceipt({ announcement: { @@ -110,6 +133,7 @@ describe('publisher contribution evidence with an independent clean control', () casRoot: cas, repositoryId: identity.repositoryId, }); + verifiedReceipts.push(verified); const cached = yield* store.cachedCommittedFileKeys(databasePath, verified.parsed.extractorSet, [ {path: verified.parsed.normalizedPath, contentHash: verified.parsed.contentHash}, ]); @@ -119,6 +143,24 @@ describe('publisher contribution evidence with an independent clean control', () yield* Fiber.interrupt(listener); yield* git(repo, ['fetch', '-q', contributor, 'main']); yield* git(repo, ['merge', '--ff-only', 'FETCH_HEAD']); + let sharingRequests = 0; + const sentinel = yield* Effect.acquireRelease( + Effect.sync(() => + Bun.serve({ + hostname: '127.0.0.1', + port: 0, + fetch: () => { + sharingRequests += 1; + return new Response('unexpected sharing request', {status: 503}); + }, + }), + ), + server => Effect.promise(() => server.stop(true)), + ); + for (const targetHome of [home, controlHome]) { + yield* writeGraphShareCoordinatorUrl(targetHome, `http://127.0.0.1:${sentinel.port}`); + yield* writeGraphShareContributionMode(targetHome, 'passive'); + } const control = yield* advanceGraphPublisherFrontier(config(controlHome), { cas: controlCas, cwd: repo, @@ -128,41 +170,126 @@ describe('publisher contribution evidence with an independent clean control', () expect(control.contributionEvidence).toMatchObject({ selectedResults: 0, verifiedResults: 0, - hydration: {status: 'completed', hydratedResults: 0}, - index: {reusedFiles: 0, totalFiles: 3}, + hydration: {status: 'skipped-source-verification', hydratedResults: 0}, + sourceUse: {consumedActions: 0, consumedResultManifestDigests: [], sourceVerifiedFiles: eligibleFiles}, + index: {reusedFiles: 0, totalFiles: eligibleFiles}, }); const publication = advanceGraphPublisherFrontier(config(home), {cas, cwd: repo, forceFreeze: true}); - let hydrationCalls = 0; - const advanced = yield* failHydration - ? publication.pipe( - Effect.provideService(CodeGraphStore, { - ...store, - cacheFacts: (...args) => - Effect.gen(function* () { - hydrationCalls += 1; - if (hydrationCalls === 2) - return yield* CodeGraphStoreError.of('synthetic hydration fault; do not expose this error'); - return yield* store.cacheFacts(...args); - }), - }), - ) - : publication; + if (scenario === 'forged') { + const sharing = graphSharingLayout(path, home, cas); + const state = yield* loadGraphShareCoordinatorState({ + organization: 'acme', + repositoryId: identity.repositoryId, + threadnoteHome: home, + }); + const original = verifiedReceipts.find(item => item.parsed.normalizedPath === 'src/a.ts')!; + const forged = graphShareParseResultArtifact({ + ...original.parsed, + facts: {...original.parsed.facts, diagnostics: ['forged but validly hashed facts']}, + }); + const resultManifestDigest = yield* putCasBytes(cas, new TextEncoder().encode(canonicalJson(forged))); + const attestationDigest = yield* putCasBytes( + cas, + new TextEncoder().encode( + canonicalJson({kind: 'contributor-self', payloadDigest: resultManifestDigest, schemaVersion: 1}), + ), + ); + const announcement = { + ...original.announcement, + resultManifestDigest, + attestationDigest, + semanticDigest: forged.semanticDigest, + }; + yield* writePrivateJsonFile(sharing.coordinatorStatePath, { + ...state, + receipts: { + ...state.receipts, + receipts: state.receipts.receipts.map(receipt => + receipt.resultManifestDigest === original.announcement.resultManifestDigest + ? announcement + : receipt, + ), + }, + }); + // All existing integrity checks pass: only independent source semantics rejects it. + yield* verifyGraphShareParseReceipt({announcement, casRoot: cas, repositoryId: identity.repositoryId}); + const pointerPath = graphSharingFrontierPointerPath(path, sharing.frontiersRoot, identity.repositoryId); + const before = yield* readJsonFile(pointerPath); + const rejected = yield* Effect.exit(publication); + expect(Exit.isFailure(rejected)).toBe(true); + if (Exit.isFailure(rejected)) + expect(Cause.pretty(rejected.cause)).toContain('Contribution source verification failed'); + expect(yield* readJsonFile(pointerPath)).toEqual(before); + // Restore the original accepted immutable receipt set. A new attempt must collect its own evidence. + yield* writePrivateJsonFile(sharing.coordinatorStatePath, state); + } + if (scenario === 'quarantine') { + const sharing = graphSharingLayout(path, home, cas); + const coordinatorOptions = { + organization: 'acme', + repositoryId: identity.repositoryId, + threadnoteHome: home, + }; + const state = yield* loadGraphShareCoordinatorState(coordinatorOptions); + const pointerPath = graphSharingFrontierPointerPath(path, sharing.frontiersRoot, identity.repositoryId); + const before = yield* readJsonFile(pointerPath); + const original = verifiedReceipts[0]; + const rejected = yield* Effect.exit( + advanceGraphPublisherFrontier(config(home), { + cas, + cwd: repo, + forceFreeze: true, + onMachine: machine => + machine.phase !== 'verifying' + ? Effect.void + : Effect.gen(function* () { + const latest = yield* loadGraphShareCoordinatorState(coordinatorOptions); + const conflict = announceGraphShareResult(latest.receipts, { + ...original.announcement, + resultManifestDigest: sha256Digest('late conflict'), + semanticDigest: sha256Digest('conflicting semantics'), + }); + expect(conflict.status).toBe('quarantined'); + yield* writePrivateJsonFile(sharing.coordinatorStatePath, { + ...latest, + receipts: conflict.store, + }); + }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.orDie, + ), + }), + ); + expect(Exit.isFailure(rejected)).toBe(true); + if (Exit.isFailure(rejected)) expect(Cause.pretty(rejected.cause)).toContain('entered quarantine'); + expect(yield* readJsonFile(pointerPath)).toEqual(before); + yield* writePrivateJsonFile(sharing.coordinatorStatePath, state); + } + const advanced = yield* publication; expect(advanced.published).toBe(true); expect(advanced.sourceCommit).toBe(target.headCommit); expect(advanced.generation).toBe(2); + if (scenario === 'abi-transition') expect(advanced.checkpointDigest).not.toBe(baseline.checkpointDigest); expect(advanced.contributionEvidence).toMatchObject({ - selectedResults: 3, - verifiedResults: 3, + selectedResults: eligibleFiles, + verifiedResults: eligibleFiles, resultDigestsTruncated: false, - hydration: failHydration - ? {status: 'failed', hydratedResults: null} - : {status: 'completed', hydratedResults: 3}, - index: {reusedFiles: 0, totalFiles: 3}, + hydration: {status: 'skipped-source-verification', hydratedResults: 0}, + sourceUse: { + consumedActions: eligibleFiles, + sourceVerifiedFiles: eligibleFiles, + resultDigestsTruncated: false, + }, + index: {reusedFiles: 0, totalFiles: eligibleFiles}, }); expect(advanced.contributionEvidence?.resultManifestDigests).toEqual( receipts.map(item => item.resultManifestDigest).sort(), ); - expect(JSON.stringify(advanced)).not.toContain('synthetic hydration fault'); + expect(advanced.contributionEvidence?.sourceUse?.consumedResultManifestDigests).toEqual( + receipts.map(item => item.resultManifestDigest).sort(), + ); // Compare whole logical graphs, independently built from the same baseline and target Git source. const actual = yield* runCodeGraphCheckpointExport(config(home), { cwd: repo, @@ -175,6 +302,8 @@ describe('publisher contribution evidence with an independent clean control', () quiet: true, }); expect(actual.logicalDigest).toBe(clean.logicalDigest); + expect(sharingRequests).toBe(0); + yield* writeGraphShareCoordinatorUrl(home, undefined); const forced = yield* indexer.index({cwd: repo, force: true, ensureVectors: false, threadnoteHome: home}); expect(forced.reusedFiles).toBe(0); const unchanged = yield* advanceGraphPublisherFrontier(config(home), {cas, cwd: repo, forceFreeze: true}); diff --git a/test/unit/code-graph.sharing-publication-evidence.test.ts b/test/unit/code-graph.sharing-publication-evidence.test.ts index 55a17b33e..d3c3f875b 100644 --- a/test/unit/code-graph.sharing-publication-evidence.test.ts +++ b/test/unit/code-graph.sharing-publication-evidence.test.ts @@ -19,6 +19,11 @@ describe('publisher contribution evidence', () => { index: {reusedFiles: 3, skippedFiles: 1, snapshot: {id: 'snapshot', fileCount: 5}}, selectedResults: digests.length, verifiedResultDigests: digests, + sourceUse: { + consumedActions: digests.length, + consumedResultManifestDigests: digests, + sourceVerifiedFiles: digests.length, + }, }; const result = graphPublisherContributionEvidence(input); expect(result.canonicalInputPolicy).toBe('publisher-recompute'); @@ -26,6 +31,9 @@ describe('publisher contribution evidence', () => { result, ); expect(digests).toEqual(before); + expect(result.sourceUse?.consumedResultManifestDigests).toEqual(result.resultManifestDigests); + expect(result.sourceUse?.resultDigestsTruncated).toBe(digests.length > 128); + expect(result.sourceUse?.consumedActions).toBe(digests.length); expect(result.resultManifestDigests.length).toBe(Math.min(128, digests.length)); expect(result.resultDigestsTruncated).toBe(digests.length > 128); expect(result.verifiedResults).toBe(digests.length); diff --git a/test/unit/code-graph.sharing-source-verification.test.ts b/test/unit/code-graph.sharing-source-verification.test.ts new file mode 100644 index 000000000..9778aa4ef --- /dev/null +++ b/test/unit/code-graph.sharing-source-verification.test.ts @@ -0,0 +1,151 @@ +import {describe, expect, it} from '@effect/vitest'; +import {Effect, Exit} from 'effect'; +import * as FC from 'effect/testing/FastCheck'; +import {serializeBoundedCodeGraphFact} from '../../src/code_graph/fact_budget.js'; +import type {CodeGraphFileFacts, CodeGraphInventoryFile} from '../../src/code_graph/types.js'; +import {graphShareParseActionKey} from '../../src/code_graph/sharing/action.js'; +import {sha256Digest} from '../../src/code_graph/sharing/digest.js'; +import {graphShareParseResultArtifact} from '../../src/code_graph/sharing/parse_result.js'; +import {makeGraphShareSourceVerification} from '../../src/code_graph/sharing/source_verification.js'; + +const repositoryId = 'a'.repeat(64), + sourceCommit = 'b'.repeat(40), + extractorSet = 'c'.repeat(64); +function fixture(path: string, diagnostics: readonly string[] = []) { + const file: CodeGraphInventoryFile = { + path, + blobId: 'd'.repeat(40), + contentHash: 'e'.repeat(64), + language: 'typescript', + mode: '100644', + source: 'commit', + size: 20, + }; + const facts: CodeGraphFileFacts = {path, symbols: [], edges: [], diagnostics}; + const parsed = graphShareParseResultArtifact({ + repositoryId, + normalizedPath: path, + gitBlobId: file.blobId, + contentHash: file.contentHash, + extractorSet, + languageAndRole: 'typescript:source', + facts, + actionKey: graphShareParseActionKey({ + repositoryId, + normalizedPath: path, + contentHash: file.contentHash, + extractorSet, + languageAndRole: 'typescript:source', + }), + }); + const item = { + parsed, + announcement: { + actionKey: parsed.actionKey, + batchId: sourceCommit, + semanticDigest: parsed.semanticDigest, + resultManifestDigest: sha256Digest(JSON.stringify(parsed)), + attestationDigest: sha256Digest('attestation'), + }, + }; + return { + file, + facts, + item, + group: {cacheIdentity: extractorSet, facts: [serializeBoundedCodeGraphFact(facts)], files: [file]}, + }; +} + +describe('source-verified original contribution assembly', () => { + it.effect.prop( + 'is order-independent and duplicate-idempotent without mutating source facts', + { + values: FC.array(FC.string({maxLength: 24}), {minLength: 1, maxLength: 12}), + }, + ({values}) => + Effect.gen(function* () { + const rows = values.map((value, i) => fixture(`src/file-${i}.ts`, [value])); + const selected = rows.filter((_, i) => i % 2 === 0); + const before = JSON.stringify(rows); + const make = () => + makeGraphShareSourceVerification({repositoryId, sourceCommit, verified: selected.map(row => row.item)}); + const left = make(), + right = make(); + for (const row of rows) yield* left.hooks.observeParserBatch(row.group); + for (const row of [...rows].reverse()) yield* right.hooks.observeParserBatch(row.group); + for (const proof of [left, right]) { + for (const row of rows) { + const batch = {facts: new Map([[row.file.path, row.facts]]), files: [row.file]}; + const first = yield* proof.hooks.materializeFacts(batch); + yield* proof.hooks.materializeFacts(batch); + expect(first.get(row.file.path)).toEqual(row.facts); + if (selected.includes(row)) expect(first.get(row.file.path)).not.toBe(row.facts); + else expect(first.get(row.file.path)).toBe(row.facts); + } + } + const evidence = yield* left.complete(); + expect(evidence).toEqual(yield* right.complete()); + expect(evidence.consumedActions).toBe(selected.length); + expect(evidence.sourceVerifiedFiles).toBe(rows.length); + expect(evidence.consumedResultManifestDigests).toEqual( + selected.map(row => row.item.announcement.resultManifestDigest).sort(), + ); + expect(JSON.stringify(rows)).toBe(before); + }), + {fastCheck: {numRuns: 30}}, + ); + + for (const changed of ['facts', 'blob', 'extractor', 'commit', 'path', 'language'] as const) { + it.effect(`rejects mismatched ${changed} despite prior receipt integrity checks`, () => + Effect.gen(function* () { + const row = fixture('src/a.ts'); + const item = { + ...row.item, + announcement: {...row.item.announcement, ...(changed === 'commit' ? {batchId: 'f'.repeat(40)} : {})}, + parsed: { + ...row.item.parsed, + ...(changed === 'facts' ? {facts: {...row.facts, diagnostics: ['forged']}} : {}), + ...(changed === 'blob' ? {gitBlobId: 'f'.repeat(40)} : {}), + ...(changed === 'extractor' ? {extractorSet: 'f'.repeat(64)} : {}), + ...(changed === 'path' ? {normalizedPath: 'src/missing.ts'} : {}), + ...(changed === 'language' ? {languageAndRole: 'javascript:source'} : {}), + }, + }; + const proof = makeGraphShareSourceVerification({repositoryId, sourceCommit, verified: [item]}); + const result = yield* Effect.exit( + Effect.gen(function* () { + yield* proof.hooks.observeParserBatch(row.group); + yield* proof.hooks.materializeFacts({facts: new Map([[row.file.path, row.facts]]), files: [row.file]}); + yield* proof.complete(); + }), + ); + expect(Exit.isFailure(result)).toBe(true); + }), + ); + } + + it.effect('rejects stale, poisoned, or unobserved cache inputs and incomplete attempts', () => + Effect.gen(function* () { + const row = fixture('src/a.ts'); + const proof = makeGraphShareSourceVerification({repositoryId, sourceCommit, verified: [row.item]}); + const batch = {facts: new Map([[row.file.path, row.facts]]), files: [row.file]}; + expect(Exit.isFailure(yield* Effect.exit(proof.hooks.materializeFacts(batch)))).toBe(true); + yield* proof.hooks.observeParserBatch(row.group); + expect(Exit.isFailure(yield* Effect.exit(proof.complete()))).toBe(true); + expect( + Exit.isFailure( + yield* Effect.exit( + proof.hooks.materializeFacts({ + ...batch, + facts: new Map([[row.file.path, {...row.facts, diagnostics: ['poison']}]]), + }), + ), + ), + ).toBe(true); + const retry = makeGraphShareSourceVerification({repositoryId, sourceCommit, verified: [row.item]}); + expect(Exit.isFailure(yield* Effect.exit(retry.hooks.materializeFacts(batch)))).toBe(true); + yield* proof.hooks.materializeFacts(batch); + expect((yield* proof.complete()).consumedActions).toBe(1); + }), + ); +});