Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/code_graph/sharing/control_server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,9 @@ const handleGraphShareHttp = (
const request = yield* HttpServerRequest.HttpServerRequest;
const pathname = requestUrlPath(request.url);
const method = request.method;
if (method === 'POST' && pathname === '/v1/assembly-leases') {
return HttpServerResponse.jsonUnsafe({error: 'publisher-operation-forbidden'}, {status: 403});
}
const casHex = parseGraphShareHttpCasPath(pathname);
if (casHex !== undefined) {
if (method === 'GET' || method === 'HEAD') return yield* serveCasBlob(options.casRoot, casHex, method === 'HEAD');
Expand Down Expand Up @@ -324,6 +327,9 @@ const receiveTag = Effect.fn('codeGraph.sharing.receiveTag')(function* (
request: HttpServerRequest.HttpServerRequest,
) {
assertGraphShareDiscoveryTag(name);
if (!name.startsWith('tn-action-')) {
return HttpServerResponse.jsonUnsafe({error: 'publisher-operation-forbidden'}, {status: 403});
}
yield* readBoundedBody(request, GRAPH_SHARE_CONTROL_MAX_BODY_BYTES);
const decoded = yield* HttpServerRequest.schemaBodyJson(GraphShareHttpTagBody, STRICT).pipe(Effect.option);
if (decoded._tag === 'None') {
Expand Down
49 changes: 49 additions & 0 deletions src/code_graph/sharing/publication_evidence.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import type {CodeGraphIndexSummary} from '../types.js';
import type {Sha256Digest} from './digest.js';

const MAX_RESULT_DIGESTS = 128;

export type GraphPublisherHydrationEvidence =
| {readonly status: 'completed'; readonly hydratedResults: number}
| {readonly status: 'failed'; readonly hydratedResults: null};

/** Local diagnostics only. Cache hydration and aggregate reuse do not prove per-worker attribution. */
export interface GraphPublisherContributionEvidence {
readonly canonicalInputPolicy: 'publisher-recompute';
readonly hydration: GraphPublisherHydrationEvidence;
readonly index: {
readonly reusedFiles: number;
readonly skippedFiles: number;
readonly snapshotId: string;
readonly totalFiles: number;
};
readonly resultDigestsTruncated: boolean;
readonly resultManifestDigests: readonly Sha256Digest[];
readonly selectedResults: number;
/** Receipt integrity and schema checks only; contributor identity and semantics are not authenticated. */
readonly verifiedResults: number;
}

export function graphPublisherContributionEvidence(input: {
readonly hydration: GraphPublisherHydrationEvidence;
readonly index: Pick<CodeGraphIndexSummary, 'reusedFiles' | 'skippedFiles'> & {
readonly snapshot: Pick<CodeGraphIndexSummary['snapshot'], 'id' | 'fileCount'>;
};
readonly selectedResults: number;
readonly verifiedResultDigests: readonly Sha256Digest[];
}): GraphPublisherContributionEvidence {
return {
canonicalInputPolicy: 'publisher-recompute',
hydration: input.hydration,
index: {
reusedFiles: input.index.reusedFiles,
skippedFiles: input.index.skippedFiles,
snapshotId: input.index.snapshot.id,
totalFiles: input.index.snapshot.fileCount,
},
resultDigestsTruncated: input.verifiedResultDigests.length > MAX_RESULT_DIGESTS,
resultManifestDigests: [...input.verifiedResultDigests].sort().slice(0, MAX_RESULT_DIGESTS),
selectedResults: input.selectedResults,
verifiedResults: input.verifiedResultDigests.length,
};
}
33 changes: 29 additions & 4 deletions src/code_graph/sharing/publisher_cycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ import {
type GraphShareProfileV1,
} from './profile.js';
import {selectGraphShareResultsForFrozenMachine} from './receipts.js';
import {
graphPublisherContributionEvidence,
type GraphPublisherContributionEvidence,
type GraphPublisherHydrationEvidence,
} from './publication_evidence.js';
import {resolveGraphShareCasRoot} from './trust.js';
import type {RepositoryIdentity} from '../types.js';

Expand All @@ -86,6 +91,7 @@ const FORCE_FREEZE_THRESHOLDS: GraphShareFrontierThresholds = {

export interface GraphPublisherAdvanceResult {
readonly checkpointDigest: Sha256Digest;
readonly contributionEvidence?: GraphPublisherContributionEvidence;
readonly descriptorDigest?: Sha256Digest;
readonly envelopeDigest: Sha256Digest;
readonly generation: number;
Expand Down Expand Up @@ -167,7 +173,7 @@ export const advanceGraphPublisherFrontier = Effect.fn('codeGraph.sharing.advanc
return currentPointer(current, pointer, machine.phase);
}
const selected = selectGraphShareResultsForFrozenMachine(coordinator.receipts, machine);
const verified = [];
const verified: VerifiedGraphShareParseReceipt[] = [];
for (const announcement of selected.selected) {
const receipt = yield* verifyGraphShareParseReceipt({
announcement,
Expand All @@ -182,11 +188,21 @@ export const advanceGraphPublisherFrontier = Effect.fn('codeGraph.sharing.advanc
}
verified.push(receipt.value);
}
yield* hydratePublisherFacts(config, identity, verified).pipe(Effect.ignore);
const hydration: GraphPublisherHydrationEvidence = yield* hydratePublisherFacts(config, identity, verified).pipe(
Effect.match({
onFailure: () => ({status: 'failed' as const, hydratedResults: null}),
onSuccess: result => ({status: 'completed' as const, hydratedResults: result.hydrated}),
}),
);
const published = yield* Effect.gen(function* () {
const indexer = yield* CodeGraphIndexer;
const store = yield* CodeGraphStore;
yield* indexer.index({cwd, ensureVectors: false, force: true, threadnoteHome: config.agentContextHome});
const indexed = yield* indexer.index({
cwd,
ensureVectors: false,
force: true,
threadnoteHome: config.agentContextHome,
});
const layout = codeGraphLayout(path, config.agentContextHome, identity.checkoutId, identity.worktreeId);
const ready = yield* store.readySnapshot(layout.databasePath, identity.worktreeId);
if (
Expand All @@ -203,7 +219,16 @@ export const advanceGraphPublisherFrontier = Effect.fn('codeGraph.sharing.advanc
yield* persistMachine(coordinatorOptions, machine, options.onMachine, options.stateRef);
machine = verifyGraphShareBatch(machine);
yield* persistMachine(coordinatorOptions, machine, options.onMachine, options.stateRef);
return yield* exportSignedGeneration(config, options, current, identity.repositoryId, profile);
const exported = yield* exportSignedGeneration(config, options, current, identity.repositoryId, profile);
return {
...exported,
contributionEvidence: graphPublisherContributionEvidence({
hydration,
index: indexed,
selectedResults: selected.selected.length,
verifiedResultDigests: verified.map(item => item.announcement.resultManifestDigest),
}),
};
}).pipe(
Effect.tapError(() => {
machine = failGraphShareBatch(machine);
Expand Down
213 changes: 213 additions & 0 deletions test/integration/code-graph.sharing-publication-evidence.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
import {describe, expect, it as effectIt} from '@effect/vitest';
import {Deferred, Effect, Fiber, FileSystem, Path} from 'effect';
import {TestClock} from 'effect/testing';
import {provideTestLayer} from '../helpers/effect-layer.js';
import {ApplicationLayer} from '../../src/effect/runtime.js';
import {runCommandEffect} from '../../src/effect/command.js';
import {CodeGraphIndexer} from '../../src/code_graph/indexer.js';
import {CodeGraphStore} from '../../src/code_graph/store.js';
import {CodeGraphStoreError} from '../../src/code_graph/types.js';
import {codeGraphLayout} from '../../src/code_graph/layout.js';
import {resolveRepositoryIdentity} from '../../src/code_graph/repository.js';
import {runCodeGraphCheckpointExport} from '../../src/code_graph/checkpoint/commands.js';
import {runGraphShareJoin} from '../../src/code_graph/sharing/client.js';
import {graphShareControlGetStatus} from '../../src/code_graph/sharing/control_client.js';
import {parseSha256Digest} from '../../src/code_graph/sharing/digest.js';
import {verifyGraphShareParseReceipt} from '../../src/code_graph/sharing/parse_cache.js';
import {advanceGraphPublisherFrontier} from '../../src/code_graph/sharing/publisher_cycle.js';
import {
runGraphPublisherBootstrap,
runGraphPublisherListen,
runGraphShareInit,
} from '../../src/code_graph/sharing/publisher.js';

describe('publisher contribution evidence with an independent clean control', () => {
for (const failHydration of [false, true]) {
effectIt.effect(
failHydration
? 'reports partial hydration failure while publishing a correct graph'
: 'distinguishes received facts from canonical publisher recomputation',
() =>
TestClock.withLive(
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const indexer = yield* CodeGraphIndexer;
const store = yield* CodeGraphStore;
const root = yield* fs.makeTempDirectoryScoped({prefix: 'threadnote-publication-evidence-'});
const repo = path.join(root, 'publisher-repo');
const contributor = path.join(root, 'contributor-repo');
const home = path.join(root, 'publisher-home');
const workerHome = path.join(root, 'worker-home');
const controlHome = path.join(root, 'control-home');
const cas = path.join(root, 'cas');
const controlCas = path.join(root, 'control-cas');
const origin = 'https://github.com/acme/publication-evidence.git';
yield* fs.makeDirectory(path.join(repo, 'src'), {recursive: true});
yield* fs.writeFileString(path.join(repo, 'package.json'), '{"private":true,"type":"module"}\n');
yield* fs.writeFileString(path.join(repo, 'src/a.ts'), 'export const alpha = 1;\n');
yield* fs.writeFileString(
path.join(repo, 'src/b.ts'),
"import {alpha} from './a.js'; export const beta = alpha + 1;\n",
);
yield* git(repo, ['init', '-q', '--initial-branch=main']);
yield* git(repo, ['remote', 'add', 'origin', origin]);
yield* commit(repo, 'baseline');
yield* runGraphShareInit(config(home), {cas, cwd: repo, organization: 'acme', writeConfig: true});
yield* commit(repo, 'enroll');
yield* indexer.index({cwd: repo, ensureVectors: false, threadnoteHome: home});
const baseline = yield* runGraphPublisherBootstrap(config(home), {cas, cwd: repo});
// Copy the same baseline before any contributor receipt or target-commit facts exist.
yield* fs.copy(home, controlHome);
yield* fs.copy(cas, controlCas);
yield* git(root, ['clone', '-q', repo, contributor]);
yield* git(contributor, ['remote', 'set-url', 'origin', origin]);
const ready = yield* Deferred.make<string>();
const listener = yield* Effect.forkScoped(
runGraphPublisherListen(config(home), {
cas,
cwd: repo,
listen: '127.0.0.1:0',
onReady: output => Deferred.succeed(ready, output.coordinatorUrl).pipe(Effect.asVoid),
}),
);
const url = yield* Deferred.await(ready);
yield* runGraphShareJoin(config(workerHome), {
cas: path.join(root, 'worker-cas'),
coordinator: url,
cwd: contributor,
});
// Every eligible source file changes, so old local parse facts cannot explain target reuse.
yield* fs.writeFileString(
path.join(contributor, 'package.json'),
'{"private":true,"type":"module","name":"target"}\n',
);
yield* fs.writeFileString(
path.join(contributor, 'src/a.ts'),
'export function alpha(value: number) { return value * 7; }\n',
);
yield* fs.writeFileString(
path.join(contributor, 'src/b.ts'),
"import {alpha} from './a.js'; export function beta(value: number) { return alpha(value) + 3; }\n",
);
yield* commit(contributor, 'target');
const target = yield* resolveRepositoryIdentity(contributor);
yield* indexer.index({cwd: contributor, ensureVectors: false, threadnoteHome: workerHome});
const status = yield* graphShareControlGetStatus(url);
const receipts = status.receipts.filter(receipt => receipt.batchId === target.headCommit);
expect(receipts).toHaveLength(3);
const identity = yield* resolveRepositoryIdentity(repo);
expect(identity.headCommit).toBe(baseline.sourceCommit);
const databasePath = codeGraphLayout(path, home, identity.checkoutId, identity.worktreeId).databasePath;
for (const announcement of receipts) {
const verified = yield* verifyGraphShareParseReceipt({
announcement: {
...announcement,
attestationDigest: parseSha256Digest(announcement.attestationDigest),
resultManifestDigest: parseSha256Digest(announcement.resultManifestDigest),
semanticDigest: parseSha256Digest(announcement.semanticDigest),
},
casRoot: cas,
repositoryId: identity.repositoryId,
});
const cached = yield* store.cachedCommittedFileKeys(databasePath, verified.parsed.extractorSet, [
{path: verified.parsed.normalizedPath, contentHash: verified.parsed.contentHash},
]);
expect(cached.size).toBe(0);
}
// Stop the watch before advancing the publisher clone; publication is now exactly controlled.
yield* Fiber.interrupt(listener);
yield* git(repo, ['fetch', '-q', contributor, 'main']);
yield* git(repo, ['merge', '--ff-only', 'FETCH_HEAD']);
const control = yield* advanceGraphPublisherFrontier(config(controlHome), {
cas: controlCas,
cwd: repo,
forceFreeze: true,
});
expect(control.published).toBe(true);
expect(control.contributionEvidence).toMatchObject({
selectedResults: 0,
verifiedResults: 0,
hydration: {status: 'completed', hydratedResults: 0},
index: {reusedFiles: 0, totalFiles: 3},
});
const publication = advanceGraphPublisherFrontier(config(home), {cas, cwd: repo, forceFreeze: true});
let hydrationCalls = 0;
const advanced = yield* failHydration
? publication.pipe(
Effect.provideService(CodeGraphStore, {
...store,
cacheFacts: (...args) =>
Effect.gen(function* () {
hydrationCalls += 1;
if (hydrationCalls === 2)
return yield* CodeGraphStoreError.of('synthetic hydration fault; do not expose this error');
return yield* store.cacheFacts(...args);
}),
}),
)
: publication;
expect(advanced.published).toBe(true);
expect(advanced.sourceCommit).toBe(target.headCommit);
expect(advanced.generation).toBe(2);
expect(advanced.contributionEvidence).toMatchObject({
selectedResults: 3,
verifiedResults: 3,
resultDigestsTruncated: false,
hydration: failHydration
? {status: 'failed', hydratedResults: null}
: {status: 'completed', hydratedResults: 3},
index: {reusedFiles: 0, totalFiles: 3},
});
expect(advanced.contributionEvidence?.resultManifestDigests).toEqual(
receipts.map(item => item.resultManifestDigest).sort(),
);
expect(JSON.stringify(advanced)).not.toContain('synthetic hydration fault');
// Compare whole logical graphs, independently built from the same baseline and target Git source.
const actual = yield* runCodeGraphCheckpointExport(config(home), {
cwd: repo,
output: path.join(root, 'actual.cgcp'),
quiet: true,
});
const clean = yield* runCodeGraphCheckpointExport(config(controlHome), {
cwd: repo,
output: path.join(root, 'clean.cgcp'),
quiet: true,
});
expect(actual.logicalDigest).toBe(clean.logicalDigest);
const forced = yield* indexer.index({cwd: repo, force: true, ensureVectors: false, threadnoteHome: home});
expect(forced.reusedFiles).toBe(0);
const unchanged = yield* advanceGraphPublisherFrontier(config(home), {cas, cwd: repo, forceFreeze: true});
expect(unchanged.published).toBe(false);
expect(unchanged.contributionEvidence).toBeUndefined();
}).pipe(provideTestLayer(ApplicationLayer)),
),
180_000,
);
}
});

function config(home: string) {
return {
account: 'local' as const,
agentContextHome: home,
agentId: 'threadnote',
manifestPath: `${home}/seed-manifest.yaml`,
user: 'local',
};
}
function git(repo: string, args: readonly string[]) {
return runCommandEffect('git', ['-C', repo, ...args]);
}
const commit = Effect.fn(function* (repo: string, message: string) {
yield* git(repo, ['add', '.']);
yield* git(repo, [
'-c',
'user.name=Threadnote Test',
'-c',
'[email protected]',
'commit',
'-qm',
message,
]);
});
Loading