diff --git a/.env.example b/.env.example index d38a884..b6814cd 100644 --- a/.env.example +++ b/.env.example @@ -13,12 +13,26 @@ GITHUB_APP_ID= GITHUB_APP_INSTALLATION_ID= GITHUB_APP_PRIVATE_KEY_PATH= GITHUB_API_BASE_URL=https://api.github.com +GITHUB_GIT_BASE_URL=https://github.com GITHUB_API_TIMEOUT_MS=15000 GITHUB_OUTBOX_RETRY_BASE_MS=5000 GITHUB_OUTBOX_MAX_ATTEMPTS=8 MAX_ACTIVE_IMPLEMENTATIONS=1 -SIMULATED_IMPLEMENTATION_MS=250 +# production is the goal/check/review path. demo is the intentionally reduced +# readiness -> worktree -> no-goal Codex -> pull-request path. +IMPLEMENTER_WORKFLOW=production MODEL_BASE_URL=http://127.0.0.1:8888/v1 MODEL_API_KEY=local READINESS_MODEL=unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL READINESS_TIMEOUT_MS=120000 +# Codex inherits the model settings above unless these are overridden. +IMPLEMENTER_MODEL_BASE_URL=http://127.0.0.1:8888/v1 +IMPLEMENTER_MODEL_API_KEY=local +IMPLEMENTER_MODEL=unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL +IMPLEMENTER_REPOSITORY_PATH=. +IMPLEMENTER_BASE_BRANCH=main +IMPLEMENTER_WORKTREE_ROOT=.data/worktrees +IMPLEMENTER_CHECK_CONFIG=.implementer.json +IMPLEMENTER_TIMEOUT_MS=1800000 +IMPLEMENTER_GIT_AUTHOR_NAME=Implementer Bot +IMPLEMENTER_GIT_AUTHOR_EMAIL=implementer@users.noreply.github.com diff --git a/.env.studio b/.env.studio index 50f6447..709fb28 100644 --- a/.env.studio +++ b/.env.studio @@ -4,4 +4,4 @@ MASTRA_DATABASE_URL=file:.data/mastra.sqlite MODEL_BASE_URL=http://127.0.0.1:8888/v1 MODEL_API_KEY=local READINESS_MODEL=unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL -READINESS_TIMEOUT_MS=120000 +READINESS_TIMEOUT_MS=600000 diff --git a/.implementer.json b/.implementer.json new file mode 100644 index 0000000..735999c --- /dev/null +++ b/.implementer.json @@ -0,0 +1,21 @@ +{ + "setup": [ + { + "name": "install dependencies", + "command": ["pnpm", "install", "--frozen-lockfile"], + "timeoutMs": 300000 + } + ], + "checks": [ + { + "name": "typecheck", + "command": ["pnpm", "typecheck"], + "timeoutMs": 120000 + }, + { + "name": "tests", + "command": ["pnpm", "test"], + "timeoutMs": 180000 + } + ] +} diff --git a/AGENTS.md b/AGENTS.md index cb329b3..5b1d941 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,6 +34,12 @@ Do not let the system review or merge its own bot-generated GitHub events. Versi ## Current implementation status -The repository currently implements GitHub ingestion, duplicate suppression, same-issue correlation, a local implementation-slot queue, Mastra persistence, the readiness half of readiness/decomposition, append-only readiness evidence, a durable GitHub clarification-comment outbox with retry and marker reconciliation, and step-aware human suspension/resumption. Only comments received after a suspension boundary may answer that suspension. GitHub question posting requires a configured GitHub App. The implementation action is still simulated. Decomposition and child-issue proposals, the real Codex goal worker, reviewer fan-out, manager, general recorder schema, pull-request integration, and the self-improvement, refactoring, and feature-suggestion scheduled graphs remain to be implemented. Do not describe those pieces as complete until code and tests exist. +The repository currently implements GitHub ingestion, duplicate suppression, same-issue correlation, a local implementation-slot queue, Mastra persistence, the readiness half of readiness/decomposition, append-only readiness evidence, a durable GitHub comment outbox with retry and marker reconciliation, and step-aware human suspension/resumption. Only comments received after a suspension boundary may answer that suspension. A real Codex SDK worker now implements accepted issues in Mastra-owned worktrees with a native persistent goal, the configured OpenAI-compatible model, automatic approval review, externally defined deterministic checks, a committed snapshot, and durable implementation evidence. Terminal implementation failures are posted to the issue and are not retried blindly. The current reviewer is deliberately an always-approve stub; successful runs can push the branch and create an idempotent pull request through a configured GitHub App, but this hosted side effect has not yet been exercised end to end in production. Decomposition and child-issue proposals, independent reviewer fan-out, manager and repair loops, the general recorder schema, human PR approval, and the self-improvement, refactoring, and feature-suggestion scheduled graphs remain to be implemented. Do not describe those pieces as complete until code and tests exist. + +An explicitly reduced `demo` workflow is also implemented for presentations. It reuses readiness and isolated-worktree preparation, runs a separate Codex worker with native goals disabled, then commits, pushes, and creates a pull request. It intentionally skips externally managed deterministic checks and independent review, and its pull-request body records that assurance boundary. `IMPLEMENTER_WORKFLOW=demo` selects it for webhook-driven runs; production remains the default. Never present a demo pull request as equivalent to a production result. When adding a node, update `docs/architecture.md`, add durable state for its inputs and outputs, and test success, retry, suspension, and duplicate-delivery behavior where applicable. + +## Code layout + +Each executable graph belongs under `src/workflows//`. Keep graph composition in `workflow.ts`. Give each meaningful Mastra step a vertical slice under `steps//`: `.definition.ts` contains its input/output schemas, inferred transition types, and Mastra definition, while `.implementation.ts` contains its workflow-specific behavior. Step definitions delegate directly to their colocated implementation. Including the step name keeps editor tabs and search results unambiguous. Additional step-specific modules may live in the same folder when an implementation grows; avoid per-step barrel files so navigation remains explicit. Do not duplicate step-transition interfaces in a central types file. Put reusable Codex, readiness/model, Git, GitHub, deterministic-check, and other technical capabilities and their contracts under `src/services/`; runtime coordination belongs under `src/orchestration/`, persistence records under `src/persistence/`, and small cross-cutting infrastructure under `src/shared/`. Keep only application composition, configuration, and executable entry points at the `src/` root. The intended navigation is `workflow.ts` to `steps//.definition.ts` to `steps//.implementation.ts` to any shared services. diff --git a/README.md b/README.md index 967705a..9cf82f4 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,25 @@ The intended system is a complete development graph: issue readiness and decomposition, Codex goal-based implementation, deterministic checks, parallel specialist reviewers, manager consolidation and human escalation, recording, and separate scheduled graphs for self-improvement, refactoring discovery, and -new-feature suggestions. The durable GitHub → Mastra loop below is the first -implemented slice, not the complete workflow. +new-feature suggestions. The durable GitHub → Mastra → Codex implementation +path below is an implemented slice, not the complete workflow. Future agent sessions should begin with `AGENTS.md`. The full design, reviewer contract, conflict handling, visual-design evidence, recorder, and learning boundary are documented in `docs/architecture.md`. Machine-readable node status -and the required reviewer set live in `src/graph-definition.ts`. +and the required reviewer set live in `src/workflows/definitions.ts`. + +The source tree follows the graph boundary. The implementation workflow is +composed in `src/workflows/implementation/workflow.ts`. Every meaningful operation +has a vertical slice under `steps//`: `.definition.ts` +contains its schemas, inferred transition types, and Mastra configuration, while +`.implementation.ts` contains its workflow behavior. Larger implementations can +be split into additional step-local modules without separating the definition +from the behavior it invokes. +Codex, Git, GitHub, checks, persistence, and process execution live in their +corresponding `src/services/`, `src/persistence/`, and `src/shared/` modules. +Planned workflows receive their own sibling directory when their first +executable step is implemented. ## How we arrived at this architecture @@ -97,7 +109,7 @@ GitHub events are recorded in SQLite first and acknowledged quickly. A coordinator gives Mastra at most one implementation slot by default. An event that arrives while Mastra is working never interrupts the current model turn. -Before the simulated implementation step, a real readiness agent calls the +Before implementation, a real readiness agent calls the OpenAI-compatible model configured in `.env`. It evaluates the issue title, body, labels, and human clarifications against a structured schema. Every attempt appends its input, decision, model ID, prompt version, graph version, timing, usage, @@ -110,6 +122,21 @@ inbox. For a different issue it creates another run, which waits for the local implementation slot. GitHub delivery IDs make retries harmless, and messages produced by the configured bot are ignored to prevent feedback loops. +For a ready issue, Mastra creates an isolated worktree and starts Codex through +the SDK. Codex is explicitly pointed at the configured OpenAI-compatible model +URL and runs with workspace write access, on-request approvals, and automatic +approval review. Its first action must create a native persistent goal; Mastra +accepts the implementation only when the recorded Codex session proves that the +goal was created and completed. Mastra, rather than Codex, owns the branch, +commit, push, and pull request. + +Repository setup and deterministic commands are declared in +`.implementer.json`. Mastra records that file before Codex starts and rejects +the result if Codex changes it. A failing setup, Codex turn, native goal, check, +commit, push, or pull-request operation stops the run and queues a durable +failure comment on the issue. Interrupted runs are also failed on restart and +their retained worktrees are not retried automatically. + ```mermaid flowchart LR GH["GitHub event"] --> IN["Durable SQLite inbox"] @@ -134,7 +161,8 @@ cp .env.example .env pnpm start ``` -The readiness agent expects an OpenAI-compatible model server. These values are +The readiness and Codex implementation agents expect an OpenAI-compatible model +server. These values are deployment configuration and belong in the ignored `.env` file: ```dotenv @@ -142,13 +170,17 @@ MODEL_BASE_URL=http://127.0.0.1:8888/v1 MODEL_API_KEY=local READINESS_MODEL=unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL READINESS_TIMEOUT_MS=120000 +IMPLEMENTER_MODEL_BASE_URL=http://127.0.0.1:8888/v1 +IMPLEMENTER_MODEL_API_KEY=local +IMPLEMENTER_MODEL=unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL ``` `READINESS_TIMEOUT_MS` prevents a stalled local generation from occupying the workflow indefinitely. No small output-token limit is imposed. The service listens only on `127.0.0.1:4317`. `GET /health` shows whether an -implementation occupies the slot, and `GET /runs` shows the durable run state. +implementation occupies the slot, `GET /runs` shows the durable run state, and +`GET /implementation-attempts` exposes implementation stages and evidence. The event database and Mastra workflow snapshots and observability traces live separately under `.data/`. Structured runtime logs are also written to the terminal. @@ -157,7 +189,8 @@ terminal. The receiver can use a GitHub App to post a readiness question directly on the issue that owns the suspended run. Create an app with repository permissions -**Metadata: read** and **Issues: read and write**, install it on the target +**Metadata: read**, **Issues: read and write**, **Contents: read and write**, and +**Pull requests: read and write**, install it on the target repository, download its private key, and place that PEM file in the ignored `.secrets/` directory. Then configure the ignored `.env` file: @@ -194,10 +227,28 @@ pnpm studio ``` Open `http://localhost:4111`, select **Workflows**, then select the -`github-implementation` workflow. Select a run in its **Runs** list to see step +`github-implementation` workflow. The reduced `github-demo-implementation` +workflow is registered alongside it. Select a run in its **Runs** list to see step status, inputs, outputs, suspension state, timing, and traces persisted in `.data/mastra.sqlite`. +### Run the reduced demo implementer + +For a short demonstration, select the additive demo workflow in the ignored +`.env` file: + +```dotenv +IMPLEMENTER_WORKFLOW=demo +``` + +Restart `pnpm start` after changing the setting. `GET /health` reports the +active `workflowMode`. The demo still performs readiness and creates an +isolated worktree, but its Codex worker has native goals disabled. Mastra then +commits, pushes, and creates the pull request without running the configured +deterministic checks or the reviewer step. The generated pull-request body +marks that reduced assurance explicitly. Remove the setting or use +`IMPLEMENTER_WORKFLOW=production` to restore the default production path. + The included [GitHub Actions workflow](.github/workflows/implementer.yml) uses a self-hosted runner labelled `implementer`. That runner opens an outbound connection to GitHub, receives the job, and posts the event to the loop service @@ -249,17 +300,27 @@ boundary, and durable GitHub comment. A subsequent human issue comment resumes the exact readiness step and triggers a new recorded evaluation. Other issues may proceed while this one waits. -## Where Codex fits today +## Current implementation boundary + +The readiness portion of `readiness-and-decomposition`, the Codex native-goal +worker, Mastra-owned worktrees, externally configured checks, implementation +evidence, local snapshot commit, and GitHub push/pull-request path are +implemented. Pull-request creation requires a configured GitHub App and remains +unverified as a hosted end-to-end operation. + +The reduced demo workflow is implemented and locally tested through worktree, +commit, and pull-request handoff with a fake publisher. Its real Codex/model and +hosted GitHub side effects have not been exercised end to end here. It is a +presentation path, not an alternative source of verification evidence. -The readiness portion of `readiness-and-decomposition` is implemented; deciding -whether to propose child issues remains planned. The implementation step still -deliberately waits for `SIMULATED_IMPLEMENTATION_MS`; this makes concurrency -deterministic in the demo and tests. It still needs to be replaced by the real -Codex goal worker. The reviewers, manager, general recorder, pull-request -integration, and all three scheduled graphs—self-improvement, refactoring -discovery, and new-feature suggestion—remain planned. Refactoring and feature -discovery only create human-reviewed proposals; approved proposals re-enter the -ordinary production graph as issues. +The reviewer step currently records an explicit always-approved stub so that the +success route can be exercised. It is not independent review and must not be +treated as evidence of quality. Decomposition, the parallel specialist +reviewers, manager and repair loop, general recorder, human PR approval, and all +three scheduled graphs—self-improvement, refactoring discovery, and new-feature +suggestion—remain planned. Refactoring and feature discovery only create +human-reviewed proposals; approved proposals re-enter the ordinary production +graph as issues. ## Verify @@ -272,5 +333,7 @@ pnpm test:readiness:live The integration tests cover the critical race: a second delivery arrives while the first Mastra run is active, attaches to that run, and does not create a second implementation. They also exercise readiness success, recorded retry, -and real Mastra suspension and resumption from a GitHub comment. The live check -is intentionally separate because it requires the configured local model. +and real Mastra suspension and resumption from a GitHub comment. They also cover +the implementation success route with a test worker and durable terminal failure +delivery. The live check is intentionally separate because it requires the +configured local model. diff --git a/docs/architecture.md b/docs/architecture.md index 94f224e..9b77e76 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,6 +2,10 @@ The webhook receiver is the entrance to the system, not the system itself. The target is a durable graph in which Codex performs one bounded implementation, independent specialists review the resulting snapshot, a manager reconciles their findings, and humans decide questions that cannot be derived from acceptance criteria or evidence. +## Code boundaries + +Every executable graph has its own directory under `src/workflows/`. The issue-to-PR implementation graph lives under `src/workflows/implementation/` and keeps composition in `workflow.ts`. Each meaningful operation has a vertical slice under `steps//`: `.definition.ts` owns its input/output schemas, schema-inferred transition types, and Mastra definition, while `.implementation.ts` owns its workflow-specific behavior. Including the step name keeps editor tabs and repository-wide search results unambiguous. A step definition delegates directly to its colocated implementation, which may then use reusable Git, Codex, GitHub, or check capabilities and contracts under `src/services/`. A growing implementation may be split into additional step-local modules; explicit imports are preferred over per-step barrel files. Persistence record shapes live with the persistence layer under `src/persistence/`; transition types are not duplicated in a central types module. Small cross-cutting infrastructure lives under `src/shared/`. This creates a deliberate navigation path from graph to definition to behavior to technical service. The self-improvement, refactoring, and feature-suggestion workflows will receive sibling directories when their executable implementations begin. + ```mermaid flowchart TD E["GitHub event"] --> I["Durable inbox"] @@ -24,11 +28,13 @@ flowchart TD Readiness is explicit because implementation must not compensate for an underspecified issue by inventing product decisions. It checks the definition of done, constraints, affected behaviour, required visual references, and available repository context. The implemented readiness agent returns a structured decision and persists every input, output, model ID, prompt version, graph version, usage record, retry, and error in an append-only evaluation table. A missing-information decision suspends the readiness step and atomically adds a question to a durable GitHub-comment outbox. When a GitHub App is configured, the outbox posts the question with a run marker, retries transient failures, and reconciles that marker before retrying so a crash after a successful API call does not duplicate the comment. A human issue comment received after the suspension boundary is added to the durable input and resumes the same Mastra run. Earlier comments cannot accidentally answer a question that did not yet exist. If the change contains multiple independently deliverable outcomes, the intended graph will propose child issues with separate acceptance criteria. That decomposition and the human-approved child-issue side effect remain planned. -The implementation node has exclusive write access to an isolated worktree. Codex uses a persistent goal containing the accepted outcome, constraints, and verification criteria. Its internal corrections and goal completion are useful but do not count as independent review. Repository-owned tests, builds, linting, type checks, and policy checks run before reviewers receive an immutable commit or diff. +The implementation node has exclusive write access to an isolated worktree. Mastra creates that worktree from the configured remote base branch and retains ownership of branching, commits, pushes, and pull requests. Codex runs through the SDK with workspace write access, on-request approvals, automatic approval review, and an explicit local OpenAI-compatible provider. Its prompt requires the native `create_goal` command before implementation and `update_goal` with `complete` only after the accepted outcome, constraints, and verification criteria are satisfied. Because the SDK event stream does not expose extension-tool calls, Mastra validates those successful native goal calls in the matching local Codex rollout before advancing. + +Repository setup and deterministic checks are externally defined in `.implementer.json`. Mastra records the check definitions before Codex starts and rejects changes to that file. Codex's internal corrections and goal completion remain useful but do not count as independent proof; Mastra runs the stored tests, builds, linting, type checks, and policy checks before committing the fixed snapshot. Any terminal failure is recorded, stops the run, and enters the durable GitHub-comment outbox. A process restart fails an interrupted run, retains its worktree for diagnosis, and never performs a blind retry. ## Review fan-out and manager -The required reviewer set lives in `src/graph-definition.ts`: security, architecture, technology/framework, performance, tests and acceptance criteria, accessibility, code quality, bug detection, and visual design. Every reviewer gets the same issue, acceptance criteria, base and candidate revisions, diff, deterministic evidence, and repository guidance. Reviewers are read-only and return structured, evidenced findings rather than modifying code. +The required reviewer set lives in `src/workflows/definitions.ts`: security, architecture, technology/framework, performance, tests and acceptance criteria, accessibility, code quality, bug detection, and visual design. Every reviewer gets the same issue, acceptance criteria, base and candidate revisions, diff, deterministic evidence, and repository guidance. Reviewers are read-only and return structured, evidenced findings rather than modifying code. The visual-design reviewer must inspect rendered evidence rather than infer appearance from source. Its inputs include baseline and candidate screenshots at agreed viewports, relevant design tokens or component examples, and important interaction states. It judges hierarchy, spacing, typography, colour, responsiveness, visual regressions, and consistency with the surrounding application. Accessibility remains separate because a coherent-looking screen can still be unusable with a keyboard or assistive technology. @@ -40,7 +46,24 @@ Repair loops are bounded. Exceeding the maximum implementation/review iterations Every delivery is persisted before acknowledgement. Duplicate delivery IDs are no-ops. An event for an active issue joins its inbox and is read at a safe boundary; it never interrupts a model turn. Different issues retain independent durable state, while a lease prevents two writers from using the same worktree. Reviewers may run in parallel because they are read-only. Bot-authored events and run markers prevent self-triggering. GitHub comment delivery is a separate durable side effect: `pending`, `sending`, `retry`, `sent`, and `failed` states plus attempt counts, API result IDs, URLs, errors, and timestamps remain inspectable in `github_comment_outbox`. -The current code implements this control-plane slice, the readiness half of readiness-and-decomposition, durable readiness evidence, durable GitHub clarification delivery, and step-aware Mastra suspension/resumption. `src/graph-definition.ts` records which nodes are implemented, partial, or planned so future sessions cannot mistake the design for completed behaviour. +The current code implements this control-plane slice, the readiness half of readiness-and-decomposition, durable readiness evidence, durable GitHub clarification delivery, step-aware Mastra suspension/resumption, the real Codex native-goal worker, Mastra-owned worktrees, deterministic checks, implementation evidence, a committed snapshot, and the GitHub push/pull-request route. The pull-request publisher first searches by head and base, making retries idempotent. The hosted side effect still requires production GitHub App validation. + +The current review node is intentionally a control-flow stub that always approves and records the warning that no independent specialist review occurred. Therefore the specialist-reviewer node is only partial, and the manager, conflict handling, and repair loop remain planned. The implementation records are a partial recorder, not the complete cross-graph experience schema. `src/workflows/definitions.ts` records which nodes are implemented, partial, or planned so future sessions cannot mistake the design for completed behaviour. + +## Reduced demo implementer + +Presentations can select a separate `github-demo-implementation` workflow. This is an additive graph; it does not remove or relax any node in the production graph. + +```mermaid +flowchart LR + R["Readiness"] --> W["Isolated worktree"] + W --> C["Codex without persistent goal"] + C --> P["Commit, push, and pull request"] +``` + +The demo reuses the durable readiness suspension/resumption path and worktree preparation. Its Codex SDK configuration disables native goals, and it does not inspect goal evidence afterward. Once Codex returns, Mastra commits the changed worktree, pushes the isolated branch, and creates an idempotent pull request. There are no externally managed deterministic checks, fixed-snapshot reviewers, manager, or repair loop in this path. The pull-request body states this lower-assurance boundary so that a demo artifact cannot be mistaken for production verification. + +Both workflows are registered with Mastra for inspection. The webhook coordinator uses the production graph by default and selects the demo graph only when `IMPLEMENTER_WORKFLOW=demo`. The active selection is visible from `GET /health`. Implementation attempts retain the model, prompt version (`codex-demo-no-goal-v1` for the demo), Codex event stream, commit, pull-request URL, and failure state. Configured check definitions may be loaded during worktree setup, but the demo never executes them and records no check results. ## Recorder and separate scheduled graphs diff --git a/package.json b/package.json index 9847224..471b0e9 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@mastra/libsql": "1.18.0", "@mastra/loggers": "1.2.0", "@mastra/observability": "1.16.3", + "@openai/codex-sdk": "^0.147.0", "zod": "^4.3.6" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e3c53b..689f803 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@mastra/observability': specifier: 1.16.3 version: 1.16.3(@mastra/core@1.55.0(express@5.2.1)(zod@4.4.3))(zod@4.4.3) + '@openai/codex-sdk': + specifier: ^0.147.0 + version: 0.147.0 zod: specifier: ^4.3.6 version: 4.4.3 @@ -569,6 +572,51 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@openai/codex-sdk@0.147.0': + resolution: {integrity: sha512-nJL0maDBZy31uEArs+u46tW22veNdHjfs96AGaFTnI3jF+g8U+a422uaPiDZwEKmyxcNwStTRz6sIh6C7XxGFQ==} + engines: {node: '>=18'} + + '@openai/codex@0.147.0': + resolution: {integrity: sha512-EQLEXecAG2ptxI7UpBMo2TR/ga5596/c/OsYF/0LoUDh5JANZ7IoGqlzBEWbuEVQ76JePIbtTW/ihCkp1a7Z3w==} + engines: {node: '>=16'} + hasBin: true + + '@openai/codex@0.147.0-darwin-arm64': + resolution: {integrity: sha512-BEUVkiOW7kLcRyrMLfAr/h9wF8sRVJyZDy6OHtVn6QGDXiv3BvAZVTY1Pu9xF7KdIdkYXbp4uayN0aDQQaAUJw==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@openai/codex@0.147.0-darwin-x64': + resolution: {integrity: sha512-Tb8McE5SvJIH0Vs5R6sq7u+quiC931yan2KOOl6km1OdZ82+Wi7eF5XrSFPs5CF7xCgoIK4Vs+byMbT5hN+ZUw==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@openai/codex@0.147.0-linux-arm64': + resolution: {integrity: sha512-SLC1JXw2TYfr/c3HhrJubyyLelq7vTOLWVmiThFA+z0+WgzCPmaseJ/kzDD3Gge/TO7fCnnj7UcPmC0d2c8XAg==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@openai/codex@0.147.0-linux-x64': + resolution: {integrity: sha512-0W9MBxPpWW0cSkNqrTDN2jR7rzzT7oNMhQY5446lT2Lw5cz5yhDTck4Va9rjkQEm+HlFzP/dmEMSZbXfJsINmw==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@openai/codex@0.147.0-win32-arm64': + resolution: {integrity: sha512-e2ZstJ8zT8Rm1nvR7CUVO+Gr3cTChE41+VfOzGhynzDXEoW0wfbjUQbc2bWbh1arG94LMm4y3dqBtUIbSrfeGA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [win32] + + '@openai/codex@0.147.0-win32-x64': + resolution: {integrity: sha512-oT7Ss5fAPf2fiWE9QNURqZcQGAAawSVxmIUdgPzckq4KFZAM+pRz9JbM4Rr498CjtbNgTOjWvDJ+DXvIBSfOPA==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + '@optimize-lodash/rollup-plugin@5.1.0': resolution: {integrity: sha512-dBQYGH8+n4Z/877e61PJteVZEc+U1wfRF6IhKqW0cABRIsqMxpWynyov6M6Sd4IUjru0dnh0EG55X86JO6WakQ==} engines: {node: '>= 18'} @@ -3028,6 +3076,37 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.20.1 + '@openai/codex-sdk@0.147.0': + dependencies: + '@openai/codex': 0.147.0 + + '@openai/codex@0.147.0': + optionalDependencies: + '@openai/codex-darwin-arm64': '@openai/codex@0.147.0-darwin-arm64' + '@openai/codex-darwin-x64': '@openai/codex@0.147.0-darwin-x64' + '@openai/codex-linux-arm64': '@openai/codex@0.147.0-linux-arm64' + '@openai/codex-linux-x64': '@openai/codex@0.147.0-linux-x64' + '@openai/codex-win32-arm64': '@openai/codex@0.147.0-win32-arm64' + '@openai/codex-win32-x64': '@openai/codex@0.147.0-win32-x64' + + '@openai/codex@0.147.0-darwin-arm64': + optional: true + + '@openai/codex@0.147.0-darwin-x64': + optional: true + + '@openai/codex@0.147.0-linux-arm64': + optional: true + + '@openai/codex@0.147.0-linux-x64': + optional: true + + '@openai/codex@0.147.0-win32-arm64': + optional: true + + '@openai/codex@0.147.0-win32-x64': + optional: true + '@optimize-lodash/rollup-plugin@5.1.0(rollup@4.62.4)': dependencies: '@optimize-lodash/transform': 3.0.6 diff --git a/scripts/check-readiness-model.ts b/scripts/check-readiness-model.ts index cc297bd..603a276 100644 --- a/scripts/check-readiness-model.ts +++ b/scripts/check-readiness-model.ts @@ -1,5 +1,6 @@ import { loadConfig } from "../src/config"; -import { createReadinessEvaluator } from "../src/readiness"; +import { createReadinessEvaluator } from "../src/services/readiness/evaluator"; +import { readinessDecisionSchema } from "../src/workflows/implementation/steps/readiness/readiness.definition"; try { process.loadEnvFile(".env"); @@ -8,7 +9,10 @@ try { } const config = loadConfig(); -const { evaluator } = createReadinessEvaluator(config.readinessModel); +const { evaluator } = createReadinessEvaluator( + config.readinessModel, + readinessDecisionSchema, +); const result = await evaluator.evaluate( { controlRunId: "readiness-live-check", diff --git a/src/app.ts b/src/app.ts index d8d35af..019bbdf 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,42 +1,75 @@ import http, { type IncomingMessage, type ServerResponse } from "node:http"; -import { normalizeGitHubEvent, verifyGitHubSignature } from "./github"; -import { GitHubCommentOutboxWorker } from "./github-comment-outbox"; +import { normalizeGitHubEvent, verifyGitHubSignature } from "./services/github/webhook"; +import { GitHubCommentOutboxWorker } from "./services/github/comment-outbox"; import { GitHubAppCommentPublisher, type GitHubCommentPublisher, -} from "./github-comments"; + type GitHubPullRequestPublisher, +} from "./services/github/client"; import type { AppConfig } from "./config"; -import { RunCoordinator } from "./coordinator"; +import { RunCoordinator } from "./orchestration/run-coordinator"; +import { createReadinessEvaluator } from "./services/readiness/evaluator"; +import type { ReadinessEvaluator } from "./services/readiness/contracts"; +import { EventStore } from "./persistence/event-store"; +import { createImplementationMastra } from "./workflows/implementation"; import { - createReadinessEvaluator, - type ReadinessEvaluator, -} from "./readiness"; -import { EventStore } from "./store"; -import { createImplementationMastra } from "./workflow"; + createImplementationWorkflowImplementations, + type ImplementationWorkflowImplementations, +} from "./workflows/implementation/create-step-implementations"; +import { + readinessDecisionSchema, + type ReadinessDecision, +} from "./workflows/implementation/steps/readiness/readiness.definition"; +import { + createDemoImplementationWorkflowImplementations, + type DemoImplementationWorkflowImplementations, +} from "./workflows/demo-implementation/create-step-implementations"; export async function createApp( config: AppConfig, options: { - readinessEvaluator?: ReadinessEvaluator; + readinessEvaluator?: ReadinessEvaluator; githubCommentPublisher?: GitHubCommentPublisher; + githubPullRequestPublisher?: GitHubPullRequestPublisher; + stepImplementations?: Partial; + demoStepImplementations?: Partial; } = {}, ) { const eventStore = new EventStore(config.databasePath); const readiness = options.readinessEvaluator ? { evaluator: options.readinessEvaluator, agent: undefined } - : createReadinessEvaluator(config.readinessModel); - const { mastra, workflow, storage } = createImplementationMastra({ + : createReadinessEvaluator(config.readinessModel, readinessDecisionSchema); + const githubAppClient = config.githubApp + ? new GitHubAppCommentPublisher(config.githubApp) + : undefined; + const githubCommentPublisher = options.githubCommentPublisher ?? githubAppClient; + const githubPullRequestPublisher = + options.githubPullRequestPublisher ?? githubAppClient ?? null; + const implementations = { + ...createImplementationWorkflowImplementations({ + config: config.implementation, + store: eventStore, + readinessEvaluator: readiness.evaluator, + publisher: githubPullRequestPublisher, + }), + ...options.stepImplementations, + }; + const demoImplementations = { + ...createDemoImplementationWorkflowImplementations({ + config: config.implementation, + store: eventStore, + readinessEvaluator: readiness.evaluator, + publisher: githubPullRequestPublisher, + }), + ...options.demoStepImplementations, + }; + const { mastra, workflow, demoWorkflow, storage } = createImplementationMastra({ databaseUrl: config.mastraDatabaseUrl, - eventStore, - readinessEvaluator: readiness.evaluator, + implementations, + demoImplementations, ...(readiness.agent ? { readinessAgent: readiness.agent } : {}), }); await storage.init(); - const githubCommentPublisher = - options.githubCommentPublisher ?? - (config.githubApp - ? new GitHubAppCommentPublisher(config.githubApp) - : undefined); const githubCommentOutbox = githubCommentPublisher ? new GitHubCommentOutboxWorker( eventStore, @@ -47,9 +80,8 @@ export async function createApp( : null; const coordinator = new RunCoordinator( eventStore, - workflow, + config.workflowMode === "demo" ? demoWorkflow : workflow, config.maxActiveImplementations, - config.simulatedImplementationMs, () => githubCommentOutbox?.wake(), ); @@ -98,6 +130,7 @@ async function route( ok: true, activeImplementations: coordinator.activeCount, githubCommentDelivery: config.githubApp ? "enabled" : "disabled", + workflowMode: config.workflowMode, pendingGitHubComments: commentOutbox.filter(({ status }) => ["pending", "sending", "retry"].includes(status), ).length, @@ -118,6 +151,13 @@ async function route( return; } + if (request.method === "GET" && request.url === "/implementation-attempts") { + sendJson(response, 200, { + attempts: store.listImplementationAttempts(), + }); + return; + } + if (request.method !== "POST" || request.url !== "/webhooks/github") { sendJson(response, 404, { error: "Not found" }); return; diff --git a/src/config.ts b/src/config.ts index 1a3700f..eb37994 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,7 @@ import path from "node:path"; export interface AppConfig { + workflowMode: "production" | "demo"; port: number; databasePath: string; mastraDatabaseUrl: string; @@ -11,12 +12,27 @@ export interface AppConfig { installationId: string; privateKeyPath: string; apiBaseUrl: string; + gitBaseUrl: string; timeoutMs: number; } | null; githubOutboxRetryBaseMs: number; githubOutboxMaxAttempts: number; maxActiveImplementations: number; - simulatedImplementationMs: number; + implementation: { + repository: string; + repositoryPath: string; + baseBranch: string; + worktreeRoot: string; + checkConfigPath: string; + timeoutMs: number; + model: { + baseUrl: string; + apiKey: string; + modelId: string; + }; + gitAuthorName: string; + gitAuthorEmail: string; + }; readinessModel: { baseUrl: string; apiKey: string; @@ -38,7 +54,13 @@ export function loadConfig( baseDirectory, ); const githubApp = loadGitHubAppConfig(env, baseDirectory); + const modelBaseUrl = env.MODEL_BASE_URL ?? "http://127.0.0.1:8888/v1"; + const modelApiKey = env.MODEL_API_KEY ?? "local"; + const readinessModelId = + env.READINESS_MODEL ?? + "unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL"; return { + workflowMode: parseWorkflowMode(env.IMPLEMENTER_WORKFLOW), port: Number(env.PORT ?? 4317), databasePath, mastraDatabaseUrl, @@ -54,21 +76,48 @@ export function loadConfig( Number(env.GITHUB_OUTBOX_MAX_ATTEMPTS ?? 8), ), maxActiveImplementations: Math.max(1, Number(env.MAX_ACTIVE_IMPLEMENTATIONS ?? 1)), - simulatedImplementationMs: Math.max( - 0, - Number(env.SIMULATED_IMPLEMENTATION_MS ?? 250), - ), + implementation: { + repository: + env.GITHUB_REPOSITORY ?? + "Soverius-AI/software-development-workflow-local-llms", + repositoryPath: path.resolve( + baseDirectory, + env.IMPLEMENTER_REPOSITORY_PATH ?? ".", + ), + baseBranch: env.IMPLEMENTER_BASE_BRANCH ?? "main", + worktreeRoot: path.resolve( + baseDirectory, + env.IMPLEMENTER_WORKTREE_ROOT ?? ".data/worktrees", + ), + checkConfigPath: env.IMPLEMENTER_CHECK_CONFIG ?? ".implementer.json", + timeoutMs: Math.max( + 1, + Number(env.IMPLEMENTER_TIMEOUT_MS ?? 30 * 60_000), + ), + model: { + baseUrl: env.IMPLEMENTER_MODEL_BASE_URL ?? modelBaseUrl, + apiKey: env.IMPLEMENTER_MODEL_API_KEY ?? modelApiKey, + modelId: env.IMPLEMENTER_MODEL ?? readinessModelId, + }, + gitAuthorName: env.IMPLEMENTER_GIT_AUTHOR_NAME ?? "Implementer Bot", + gitAuthorEmail: + env.IMPLEMENTER_GIT_AUTHOR_EMAIL ?? "implementer@users.noreply.github.com", + }, readinessModel: { - baseUrl: env.MODEL_BASE_URL ?? "http://127.0.0.1:8888/v1", - apiKey: env.MODEL_API_KEY ?? "local", - modelId: - env.READINESS_MODEL ?? - "unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL", + baseUrl: modelBaseUrl, + apiKey: modelApiKey, + modelId: readinessModelId, timeoutMs: Math.max(1, Number(env.READINESS_TIMEOUT_MS ?? 120_000)), }, }; } +function parseWorkflowMode(value: string | undefined): AppConfig["workflowMode"] { + if (!value || value === "production") return "production"; + if (value === "demo") return "demo"; + throw new Error("IMPLEMENTER_WORKFLOW must be either production or demo."); +} + function loadGitHubAppConfig( env: NodeJS.ProcessEnv, baseDirectory: string, @@ -90,6 +139,10 @@ function loadGitHubAppConfig( /\/$/, "", ), + gitBaseUrl: (env.GITHUB_GIT_BASE_URL ?? "https://github.com").replace( + /\/$/, + "", + ), timeoutMs: Math.max(1, Number(env.GITHUB_API_TIMEOUT_MS ?? 15_000)), }; } diff --git a/src/mastra/index.ts b/src/mastra/index.ts index fe8eced..b47fe6b 100644 --- a/src/mastra/index.ts +++ b/src/mastra/index.ts @@ -1,7 +1,11 @@ import { loadConfig } from "../config"; -import { EventStore } from "../store"; -import { createReadinessEvaluator } from "../readiness"; -import { createImplementationMastra } from "../workflow"; +import { EventStore } from "../persistence/event-store"; +import { createReadinessEvaluator } from "../services/readiness/evaluator"; +import { createImplementationMastra } from "../workflows/implementation"; +import { GitHubAppCommentPublisher } from "../services/github/client"; +import { createImplementationWorkflowImplementations } from "../workflows/implementation/create-step-implementations"; +import { readinessDecisionSchema } from "../workflows/implementation/steps/readiness/readiness.definition"; +import { createDemoImplementationWorkflowImplementations } from "../workflows/demo-implementation/create-step-implementations"; try { process.loadEnvFile(".env"); @@ -12,11 +16,29 @@ try { const projectRoot = process.env.INIT_CWD ?? process.env.PWD ?? process.cwd(); const config = loadConfig(process.env, projectRoot); const eventStore = new EventStore(config.databasePath); -const readiness = createReadinessEvaluator(config.readinessModel); +const readiness = createReadinessEvaluator( + config.readinessModel, + readinessDecisionSchema, +); +const githubAppClient = config.githubApp + ? new GitHubAppCommentPublisher(config.githubApp) + : null; +const implementations = createImplementationWorkflowImplementations({ + config: config.implementation, + store: eventStore, + readinessEvaluator: readiness.evaluator, + publisher: githubAppClient, +}); +const demoImplementations = createDemoImplementationWorkflowImplementations({ + config: config.implementation, + store: eventStore, + readinessEvaluator: readiness.evaluator, + publisher: githubAppClient, +}); export const { mastra } = createImplementationMastra({ databaseUrl: config.mastraDatabaseUrl, - eventStore, - readinessEvaluator: readiness.evaluator, + implementations, + demoImplementations, readinessAgent: readiness.agent, }); diff --git a/src/coordinator.ts b/src/orchestration/run-coordinator.ts similarity index 91% rename from src/coordinator.ts rename to src/orchestration/run-coordinator.ts index 82c93d2..f5cbf3a 100644 --- a/src/coordinator.ts +++ b/src/orchestration/run-coordinator.ts @@ -1,6 +1,6 @@ import type { Workflow } from "@mastra/core/workflows"; -import type { EventStore } from "./store"; -import type { WorkflowRunRecord } from "./types"; +import type { WorkflowRunRecord } from "../persistence/records"; +import type { EventStore } from "../persistence/event-store"; export class RunCoordinator { private active = 0; @@ -12,7 +12,6 @@ export class RunCoordinator { private readonly store: EventStore, private readonly workflow: Workflow, private readonly maxActive: number, - private readonly implementationMs: number, private readonly onGitHubCommentQueued: () => void = () => {}, ) {} @@ -83,16 +82,14 @@ export class RunCoordinator { inputData: { controlRunId: record.id, correlationKey: record.correlationKey, - implementationMs: this.implementationMs, queuedEventsSeen: 0, }, }); this.finish(record.id, result); } catch (error) { - this.store.setStatus(record.id, "failed", { - error: error instanceof Error ? error.message : String(error), - }); + const queued = this.store.failRunAndEnqueueComment(record.id, error); + if (queued) this.onGitHubCommentQueued(); } } @@ -137,9 +134,11 @@ export class RunCoordinator { if (queued) this.onGitHubCommentQueued(); return; } - this.store.setStatus(recordId, "failed", { - error: result.error?.message ?? `Mastra ended with ${result.status}`, - }); + const queued = this.store.failRunAndEnqueueComment( + recordId, + result.error?.message ?? `Mastra ended with ${result.status}`, + ); + if (queued) this.onGitHubCommentQueued(); } } diff --git a/src/store.ts b/src/persistence/event-store.ts similarity index 65% rename from src/store.ts rename to src/persistence/event-store.ts index 2fef845..eccd78e 100644 --- a/src/store.ts +++ b/src/persistence/event-store.ts @@ -5,13 +5,19 @@ import { DatabaseSync } from "node:sqlite"; import type { IngestResult, GitHubCommentOutboxRecord, - NormalizedGitHubEvent, + ImplementationAttemptRecord, + ImplementationStage, ReadinessEvaluationRecord, ReadinessEvaluationResult, - ReadinessInput, RunStatus, WorkflowRunRecord, -} from "./types"; +} from "./records"; +import type { + ImplementationCheckDefinition, + ImplementationCheckResult, +} from "../services/checks/contracts"; +import type { NormalizedGitHubEvent } from "../services/github/contracts"; +import type { ReadinessInput } from "../services/readiness/contracts"; const ACTIVE_STATUSES = "'queued','running','waiting_human'"; @@ -110,6 +116,42 @@ export class EventStore { ); CREATE INDEX IF NOT EXISTS github_comment_outbox_due ON github_comment_outbox(status, next_attempt_at); + + CREATE TABLE IF NOT EXISTS implementation_attempts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL REFERENCES workflow_runs(id), + attempt INTEGER NOT NULL, + status TEXT NOT NULL, + stage TEXT NOT NULL, + repository_path TEXT NOT NULL, + worktree_path TEXT NOT NULL, + branch TEXT NOT NULL, + base_sha TEXT, + codex_thread_id TEXT, + goal TEXT NOT NULL, + model_base_url TEXT NOT NULL, + model_id TEXT NOT NULL, + prompt_version TEXT NOT NULL, + checks_json TEXT NOT NULL DEFAULT '[]', + check_results_json TEXT, + final_response TEXT, + review_json TEXT, + commit_sha TEXT, + pull_request_url TEXT, + error TEXT, + created_at TEXT NOT NULL, + completed_at TEXT, + UNIQUE(run_id, attempt) + ); + + CREATE TABLE IF NOT EXISTS implementation_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + attempt_id INTEGER NOT NULL REFERENCES implementation_attempts(id), + event_json TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS implementation_events_attempt + ON implementation_events(attempt_id, id); `); const runColumns = this.db .prepare("PRAGMA table_info(workflow_runs)") @@ -186,6 +228,48 @@ export class EventStore { WHERE status = 'sending'`, ) .run(now, now); + this.recoverInterruptedRuns(now); + } + + private recoverInterruptedRuns(now: string): void { + const interrupted = this.db + .prepare( + `SELECT id, repository, issue_number FROM workflow_runs + WHERE status = 'running'`, + ) + .all() as Array<{ + id: string; + repository: string; + issue_number: number | null; + }>; + for (const run of interrupted) { + const error = + "The implementer process stopped while this run was active. Its worktree was retained for diagnosis; the run was not retried automatically."; + this.db + .prepare( + `UPDATE workflow_runs SET status = 'failed', last_error = ?, updated_at = ? + WHERE id = ?`, + ) + .run(error, now, run.id); + this.db + .prepare( + `UPDATE implementation_attempts + SET status = 'error', error = ?, completed_at = ? + WHERE run_id = ? AND status = 'running'`, + ) + .run(error, now, run.id); + if (run.issue_number !== null) { + const marker = ``; + this.insertGitHubCommentOutbox( + run.id, + run.repository, + run.issue_number, + formatImplementationFailureComment("interrupted", error, marker), + marker, + now, + ); + } + } } ingest(event: NormalizedGitHubEvent, botLogin: string): IngestResult { @@ -505,6 +589,226 @@ export class EventStore { })); } + startImplementationAttempt(input: { + runId: string; + repositoryPath: string; + worktreePath: string; + branch: string; + goal: string; + modelBaseUrl: string; + modelId: string; + promptVersion: string; + }): number { + const attemptRow = this.db + .prepare( + `SELECT COALESCE(MAX(attempt), 0) + 1 AS attempt + FROM implementation_attempts WHERE run_id = ?`, + ) + .get(input.runId) as { attempt: number }; + const result = this.db + .prepare( + `INSERT INTO implementation_attempts + (run_id, attempt, status, stage, repository_path, worktree_path, + branch, goal, model_base_url, model_id, prompt_version, created_at) + VALUES (?, ?, 'running', 'preparing', ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + input.runId, + attemptRow.attempt, + input.repositoryPath, + input.worktreePath, + input.branch, + input.goal, + input.modelBaseUrl, + input.modelId, + input.promptVersion, + new Date().toISOString(), + ); + return Number(result.lastInsertRowid); + } + + configureImplementationAttempt( + id: number, + baseSha: string, + checks: ImplementationCheckDefinition[], + ): void { + this.db + .prepare( + `UPDATE implementation_attempts SET base_sha = ?, checks_json = ? + WHERE id = ?`, + ) + .run(baseSha, JSON.stringify(checks), id); + } + + setImplementationStage(id: number, stage: ImplementationStage): void { + this.db + .prepare( + `UPDATE implementation_attempts SET stage = ? + WHERE id = ? AND status = 'running'`, + ) + .run(stage, id); + } + + setCodexThreadId(id: number, threadId: string): void { + this.db + .prepare( + "UPDATE implementation_attempts SET codex_thread_id = ? WHERE id = ?", + ) + .run(threadId, id); + } + + appendImplementationEvent(id: number, event: unknown): void { + this.db + .prepare( + `INSERT INTO implementation_events (attempt_id, event_json, created_at) + VALUES (?, ?, ?)`, + ) + .run(id, JSON.stringify(event), new Date().toISOString()); + } + + completeCodexImplementation(id: number, finalResponse: string): void { + this.db + .prepare( + "UPDATE implementation_attempts SET final_response = ? WHERE id = ?", + ) + .run(finalResponse, id); + } + + recordImplementationCheck( + id: number, + result: ImplementationCheckResult, + ): void { + const row = this.db + .prepare("SELECT check_results_json FROM implementation_attempts WHERE id = ?") + .get(id) as { check_results_json: string | null } | undefined; + if (!row) throw new Error(`Implementation attempt ${id} does not exist.`); + const results = row.check_results_json + ? (JSON.parse(row.check_results_json) as ImplementationCheckResult[]) + : []; + results.push(result); + this.db + .prepare( + "UPDATE implementation_attempts SET check_results_json = ? WHERE id = ?", + ) + .run(JSON.stringify(results), id); + } + + recordImplementationSnapshot(id: number, commitSha: string): void { + this.db + .prepare("UPDATE implementation_attempts SET commit_sha = ? WHERE id = ?") + .run(commitSha, id); + } + + recordStubReview(id: number, review: unknown): void { + this.db + .prepare("UPDATE implementation_attempts SET review_json = ? WHERE id = ?") + .run(JSON.stringify(review), id); + } + + completeImplementationAttempt(id: number, pullRequestUrl: string): void { + this.db + .prepare( + `UPDATE implementation_attempts + SET status = 'success', stage = 'completed', pull_request_url = ?, + error = NULL, completed_at = ? WHERE id = ?`, + ) + .run(pullRequestUrl, new Date().toISOString(), id); + } + + failImplementationAttempt(id: number, error: unknown): void { + this.db + .prepare( + `UPDATE implementation_attempts + SET status = 'error', error = ?, completed_at = ? + WHERE id = ? AND status = 'running'`, + ) + .run(errorMessage(error), new Date().toISOString(), id); + } + + getImplementationAttempt(id: number): ImplementationAttemptRecord | null { + const row = this.db + .prepare("SELECT * FROM implementation_attempts WHERE id = ?") + .get(id) as Record | undefined; + return row ? this.mapImplementationAttempt(row) : null; + } + + getLatestImplementationAttempt(runId: string): ImplementationAttemptRecord | null { + const row = this.db + .prepare( + `SELECT * FROM implementation_attempts + WHERE run_id = ? ORDER BY attempt DESC LIMIT 1`, + ) + .get(runId) as Record | undefined; + return row ? this.mapImplementationAttempt(row) : null; + } + + listImplementationAttempts(runId?: string): ImplementationAttemptRecord[] { + const rows = (runId + ? this.db + .prepare( + "SELECT * FROM implementation_attempts WHERE run_id = ? ORDER BY attempt", + ) + .all(runId) + : this.db + .prepare("SELECT * FROM implementation_attempts ORDER BY id") + .all()) as Record[]; + return rows.map((row) => this.mapImplementationAttempt(row)); + } + + failRunAndEnqueueComment(runId: string, error: unknown): boolean { + const now = new Date().toISOString(); + const message = errorMessage(error); + this.db.exec("BEGIN IMMEDIATE"); + try { + const run = this.db + .prepare( + "SELECT repository, issue_number FROM workflow_runs WHERE id = ?", + ) + .get(runId) as + | { repository: string; issue_number: number | null } + | undefined; + if (!run) throw new Error(`Workflow run ${runId} does not exist.`); + const attempt = this.db + .prepare( + `SELECT stage FROM implementation_attempts + WHERE run_id = ? ORDER BY attempt DESC LIMIT 1`, + ) + .get(runId) as { stage: string } | undefined; + this.db + .prepare( + `UPDATE workflow_runs SET status = 'failed', pending_step = NULL, + pending_question = NULL, answer_after_event_id = NULL, + last_error = ?, updated_at = ? WHERE id = ?`, + ) + .run(message, now, runId); + this.db + .prepare( + `UPDATE implementation_attempts + SET status = 'error', error = ?, completed_at = ? + WHERE run_id = ? AND status = 'running'`, + ) + .run(message, now, runId); + let queued = false; + if (run.issue_number !== null) { + const stage = attempt?.stage ?? "starting"; + const marker = ``; + queued = this.insertGitHubCommentOutbox( + runId, + run.repository, + run.issue_number, + formatImplementationFailureComment(stage, message, marker), + marker, + now, + ); + } + this.db.exec("COMMIT"); + return queued; + } catch (caught) { + this.db.exec("ROLLBACK"); + throw caught; + } + } + getRun(id: string): WorkflowRunRecord | null { const row = this.db .prepare("SELECT * FROM workflow_runs WHERE id = ?") @@ -745,6 +1049,72 @@ export class EventStore { .run(new Date().toISOString(), runId); } + private insertGitHubCommentOutbox( + runId: string, + repository: string, + issueNumber: number, + body: string, + marker: string, + now: string, + ): boolean { + const result = this.db + .prepare( + `INSERT OR IGNORE INTO github_comment_outbox + (run_id, repository, issue_number, body, marker, status, attempts, + next_attempt_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?)`, + ) + .run( + runId, + repository, + issueNumber, + body, + marker, + now, + now, + now, + ); + return result.changes === 1; + } + + private mapImplementationAttempt( + row: Record, + ): ImplementationAttemptRecord { + return { + id: Number(row.id), + runId: String(row.run_id), + attempt: Number(row.attempt), + status: String(row.status) as ImplementationAttemptRecord["status"], + stage: String(row.stage) as ImplementationStage, + repositoryPath: String(row.repository_path), + worktreePath: String(row.worktree_path), + branch: String(row.branch), + baseSha: row.base_sha === null ? null : String(row.base_sha), + codexThreadId: + row.codex_thread_id === null ? null : String(row.codex_thread_id), + goal: String(row.goal), + modelBaseUrl: String(row.model_base_url), + modelId: String(row.model_id), + promptVersion: String(row.prompt_version), + checks: JSON.parse(String(row.checks_json)) as ImplementationCheckDefinition[], + checkResults: + row.check_results_json === null + ? null + : (JSON.parse(String(row.check_results_json)) as ImplementationCheckResult[]), + finalResponse: + row.final_response === null ? null : String(row.final_response), + review: + row.review_json === null ? null : JSON.parse(String(row.review_json)), + commitSha: row.commit_sha === null ? null : String(row.commit_sha), + pullRequestUrl: + row.pull_request_url === null ? null : String(row.pull_request_url), + error: row.error === null ? null : String(row.error), + createdAt: String(row.created_at), + completedAt: + row.completed_at === null ? null : String(row.completed_at), + }; + } + private mapRun(row: Record): WorkflowRunRecord { return { id: String(row.id), @@ -791,3 +1161,28 @@ export class EventStore { }; } } + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function formatImplementationFailureComment( + stage: string, + error: string, + marker: string, +): string { + const summary = error.length > 2_000 ? `${error.slice(0, 2_000)}…` : error; + return `### Implementation stopped + +The run could not complete during **${stage}**. + +\`\`\`text +${summary.replaceAll("```", "''' ")} +\`\`\` + +The worktree was retained for diagnosis. The run was not retried automatically. + +Run ID: \`${marker.match(/mastra-loop:([^:]+)/)?.[1] ?? "unknown"}\` + +${marker}`; +} diff --git a/src/types.ts b/src/persistence/records.ts similarity index 54% rename from src/types.ts rename to src/persistence/records.ts index b983e41..b326442 100644 --- a/src/types.ts +++ b/src/persistence/records.ts @@ -1,3 +1,13 @@ +import type { + ImplementationCheckDefinition, + ImplementationCheckResult, +} from "../services/checks/contracts"; +import type { + ReadinessEvaluation, + ReadinessInput, +} from "../services/readiness/contracts"; +import type { ReadinessDecision } from "../workflows/implementation/steps/readiness/readiness.definition"; + export type RunStatus = | "queued" | "running" @@ -5,20 +15,6 @@ export type RunStatus = | "completed" | "failed"; -export interface NormalizedGitHubEvent { - deliveryId: string; - eventName: string; - action: string | null; - repository: string; - correlationKey: string; - issueNumber: number | null; - senderLogin: string | null; - isHumanComment: boolean; - commentBody: string | null; - requiresHuman: boolean; - payload: unknown; -} - export interface WorkflowRunRecord { id: string; correlationKey: string; @@ -52,33 +48,7 @@ export interface GitHubCommentOutboxRecord { sentAt: string | null; } -export interface ReadinessInput { - controlRunId: string; - correlationKey: string; - repository: string; - issueNumber: number | null; - title: string; - body: string; - labels: string[]; - clarifications: string[]; -} - -export interface ReadinessDecision { - ready: boolean; - summary: string; - acceptanceCriteria: string[]; - missingInformation: string[]; - question: string | null; -} - -export interface ReadinessEvaluationResult { - decision: ReadinessDecision; - modelId: string; - promptVersion: string; - traceId: string | null; - finishReason: string | null; - usage: unknown; -} +export type ReadinessEvaluationResult = ReadinessEvaluation; export interface ReadinessEvaluationRecord { id: number; @@ -95,6 +65,41 @@ export interface ReadinessEvaluationRecord { completedAt: string | null; } +export type ImplementationStage = + | "preparing" + | "implementing" + | "checking" + | "snapshotting" + | "reviewing" + | "publishing" + | "completed"; + +export interface ImplementationAttemptRecord { + id: number; + runId: string; + attempt: number; + status: "running" | "success" | "error"; + stage: ImplementationStage; + repositoryPath: string; + worktreePath: string; + branch: string; + baseSha: string | null; + codexThreadId: string | null; + goal: string; + modelBaseUrl: string; + modelId: string; + promptVersion: string; + checks: ImplementationCheckDefinition[]; + checkResults: ImplementationCheckResult[] | null; + finalResponse: string | null; + review: unknown; + commitSha: string | null; + pullRequestUrl: string | null; + error: string | null; + createdAt: string; + completedAt: string | null; +} + export interface IngestResult { outcome: "created" | "attached" | "resume_requested" | "duplicate" | "ignored"; runId: string | null; diff --git a/src/services/checks/configuration.ts b/src/services/checks/configuration.ts new file mode 100644 index 0000000..7ac5cb8 --- /dev/null +++ b/src/services/checks/configuration.ts @@ -0,0 +1,30 @@ +import fs from "node:fs"; +import { z } from "zod"; +import type { ImplementationCheckDefinition } from "./contracts"; + +const commandSchema = z.object({ + name: z.string().min(1), + command: z.array(z.string().min(1)).min(1), + timeoutMs: z.number().int().positive(), +}); + +const repositoryConfigSchema = z.object({ + setup: z.array(commandSchema).default([]), + checks: z.array(commandSchema).min(1), +}); + +export interface RepositoryCheckConfiguration { + setup: ImplementationCheckDefinition[]; + checks: ImplementationCheckDefinition[]; +} + +export function loadRepositoryCheckConfiguration( + configPath: string, +): RepositoryCheckConfiguration { + if (!fs.existsSync(configPath)) { + throw new Error(`Repository check configuration does not exist: ${configPath}`); + } + return repositoryConfigSchema.parse( + JSON.parse(fs.readFileSync(configPath, "utf8")), + ); +} diff --git a/src/services/checks/contracts.ts b/src/services/checks/contracts.ts new file mode 100644 index 0000000..35538a5 --- /dev/null +++ b/src/services/checks/contracts.ts @@ -0,0 +1,13 @@ +export interface ImplementationCheckDefinition { + name: string; + command: string[]; + timeoutMs: number; +} + +export interface ImplementationCheckResult { + name: string; + command: string[]; + exitCode: number; + output: string; + durationMs: number; +} diff --git a/src/services/checks/runner.ts b/src/services/checks/runner.ts new file mode 100644 index 0000000..811b4fb --- /dev/null +++ b/src/services/checks/runner.ts @@ -0,0 +1,59 @@ +import type { EventStore } from "../../persistence/event-store"; +import type { + ImplementationCheckDefinition, + ImplementationCheckResult, +} from "./contracts"; +import { runCheckedProcess, runProcess } from "../../shared/process"; + +export async function runSetupCommands(input: { + commands: ImplementationCheckDefinition[]; + attemptId: number; + worktreePath: string; + signal: AbortSignal; + store: EventStore; +}): Promise { + for (const setup of input.commands) { + const [command, ...args] = setup.command; + if (!command) throw new Error(`Setup command ${setup.name} is empty.`); + const result = await runCheckedProcess(command, args, { + cwd: input.worktreePath, + timeoutMs: setup.timeoutMs, + signal: input.signal, + }); + input.store.appendImplementationEvent(input.attemptId, { + type: "setup.completed", + name: setup.name, + command: setup.command, + durationMs: result.durationMs, + output: result.output, + }); + } +} + +export async function runDeterministicChecks(input: { + checks: ImplementationCheckDefinition[]; + attemptId: number; + worktreePath: string; + signal: AbortSignal; + store: EventStore; +}): Promise { + const results: ImplementationCheckResult[] = []; + for (const check of input.checks) { + const [command, ...args] = check.command; + if (!command) throw new Error(`Check ${check.name} has an empty command.`); + const result = await runProcess(command, args, { + cwd: input.worktreePath, + timeoutMs: check.timeoutMs, + signal: input.signal, + }); + const checkResult = { ...check, ...result }; + results.push(checkResult); + input.store.recordImplementationCheck(input.attemptId, checkResult); + if (result.exitCode !== 0) { + throw new Error( + `Deterministic check ${check.name} failed with exit code ${result.exitCode}.\n${result.output}`, + ); + } + } + return results; +} diff --git a/src/services/codex/demo-worker.ts b/src/services/codex/demo-worker.ts new file mode 100644 index 0000000..c309a20 --- /dev/null +++ b/src/services/codex/demo-worker.ts @@ -0,0 +1,68 @@ +import { Codex } from "@openai/codex-sdk"; +import type { AppConfig } from "../../config"; +import type { EventStore } from "../../persistence/event-store"; +import { formatDemoImplementationPrompt } from "./prompts"; + +export async function runDemoCodexWorker(input: { + prepared: { + attemptId: number; + worktreePath: string; + goal: string; + }; + signal: AbortSignal; + config: AppConfig["implementation"]; + store: EventStore; +}): Promise<{ codexThreadId: string; finalResponse: string }> { + const { prepared, config, store } = input; + const codex = new Codex({ + apiKey: config.model.apiKey, + config: { + model_provider: "implementer_local", + model_providers: { + implementer_local: { + name: "Configured local demo implementer", + base_url: config.model.baseUrl, + wire_api: "responses", + requires_openai_auth: false, + }, + }, + approvals_reviewer: "auto_review", + features: { goals: false }, + web_search: "disabled", + }, + }); + const thread = codex.startThread({ + model: config.model.modelId, + sandboxMode: "workspace-write", + approvalPolicy: "on-request", + workingDirectory: prepared.worktreePath, + networkAccessEnabled: false, + }); + const signal = AbortSignal.any([ + input.signal, + AbortSignal.timeout(config.timeoutMs), + ]); + const prompt = formatDemoImplementationPrompt( + prepared.goal, + config.checkConfigPath, + ); + const { events } = await thread.runStreamed(prompt, { signal }); + let finalResponse = ""; + let threadId: string | null = null; + let turnFailure: string | null = null; + + for await (const event of events) { + store.appendImplementationEvent(prepared.attemptId, event); + if (event.type === "thread.started") { + threadId = event.thread_id; + store.setCodexThreadId(prepared.attemptId, threadId); + } + if (event.type === "item.completed" && event.item.type === "agent_message") { + finalResponse = event.item.text; + } + if (event.type === "turn.failed") turnFailure = event.error.message; + } + if (turnFailure) throw new Error(turnFailure); + if (!threadId) throw new Error("Codex did not return a thread ID."); + return { codexThreadId: threadId, finalResponse }; +} diff --git a/src/services/codex/goal-evidence.ts b/src/services/codex/goal-evidence.ts new file mode 100644 index 0000000..8bc95d9 --- /dev/null +++ b/src/services/codex/goal-evidence.ts @@ -0,0 +1,69 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +export interface NativeGoalEvidence { + created: boolean; + completed: boolean; + rolloutPath: string | null; +} + +export function readNativeGoalEvidence(threadId: string): NativeGoalEvidence { + const codexHome = process.env.CODEX_HOME ?? path.join(os.homedir(), ".codex"); + const sessionsRoot = path.join(codexHome, "sessions"); + if (!fs.existsSync(sessionsRoot)) { + return { created: false, completed: false, rolloutPath: null }; + } + const rolloutPath = fs + .globSync("**/*.jsonl", { cwd: sessionsRoot }) + .map((candidate) => path.join(sessionsRoot, candidate)) + .find((candidate) => candidate.endsWith(`-${threadId}.jsonl`)); + if (!rolloutPath) { + return { created: false, completed: false, rolloutPath: null }; + } + + const createCallIds = new Set(); + const completeCallIds = new Set(); + const successfulCallIds = new Set(); + for (const line of fs.readFileSync(rolloutPath, "utf8").split("\n")) { + if (!line) continue; + let entry: any; + try { + entry = JSON.parse(line); + } catch { + continue; + } + const payload = entry?.payload; + if (entry?.type !== "response_item") continue; + if (payload?.type === "function_call") { + if (payload.name === "create_goal") createCallIds.add(payload.call_id); + if (payload.name === "update_goal") { + try { + if (JSON.parse(payload.arguments)?.status === "complete") { + completeCallIds.add(payload.call_id); + } + } catch { + // An invalid tool call cannot establish completion. + } + } + } + if (payload?.type === "function_call_output") { + try { + const output = JSON.parse(payload.output); + if ( + output?.goal?.status === "active" || + output?.goal?.status === "complete" + ) { + successfulCallIds.add(payload.call_id); + } + } catch { + // An invalid tool result cannot establish goal state. + } + } + } + return { + created: [...createCallIds].some((id) => successfulCallIds.has(id)), + completed: [...completeCallIds].some((id) => successfulCallIds.has(id)), + rolloutPath, + }; +} diff --git a/src/services/codex/prompts.ts b/src/services/codex/prompts.ts new file mode 100644 index 0000000..b0c5ebf --- /dev/null +++ b/src/services/codex/prompts.ts @@ -0,0 +1,54 @@ +export const IMPLEMENTATION_PROMPT_VERSION = "codex-goal-v1"; +export const DEMO_IMPLEMENTATION_PROMPT_VERSION = "codex-demo-no-goal-v1"; + +export function formatGoal(input: { + title: string; + body: string; + clarifications: string[]; + acceptanceCriteria: string[]; +}): string { + const criteria = input.acceptanceCriteria.map((item) => `- ${item}`).join("\n"); + const clarifications = input.clarifications.length + ? input.clarifications.map((item) => `- ${item}`).join("\n") + : "- None"; + return `Implement the accepted GitHub issue completely in the assigned worktree. + +Issue: ${input.title} + +Description: +${input.body} + +Human clarifications: +${clarifications} + +Acceptance criteria: +${criteria} + +Preserve repository guidance and scope. Produce the code and tests needed to satisfy every acceptance criterion. Do not push, create a pull request, change branches, or mark the goal complete until the implementation is genuinely finished and verified.`; +} + +export function formatImplementationPrompt( + goal: string, + checkConfigPath: string, +): string { + return `You are the single write-capable implementation worker. + +Your first action must be to call the native create_goal command with the full objective below. Do not set a token budget. Do not begin implementation before the goal exists. Continue working against that native goal until every acceptance criterion is satisfied. Call update_goal with status complete only after current evidence proves the entire objective is complete. + +${goal} + +Mastra owns the worktree, branch, commit, push, and pull request. Do not perform those Git operations. Do not edit ${checkConfigPath}; its commands are the independent deterministic verification boundary. You may run focused checks for self-correction, but Mastra will run the recorded checks separately after your goal completes. If an eligible permission request is necessary, use the normal approval mechanism; automatic review is configured.`; +} + +export function formatDemoImplementationPrompt( + objective: string, + checkConfigPath: string, +): string { + return `You are the single write-capable implementation worker in a deliberately reduced-assurance demo workflow. + +${objective} + +Implement the requested change completely in the assigned worktree. Do not create or update a native Codex goal. Preserve repository guidance and scope, inspect your work, and run focused checks when useful for self-correction. + +Mastra owns the worktree, branch, commit, push, and pull request. Do not perform those Git operations. Do not edit ${checkConfigPath}. This demo workflow will publish your result without externally managed deterministic checks or independent review, so state any verification you performed accurately in your final response.`; +} diff --git a/src/services/codex/worker.ts b/src/services/codex/worker.ts new file mode 100644 index 0000000..b680292 --- /dev/null +++ b/src/services/codex/worker.ts @@ -0,0 +1,81 @@ +import { Codex } from "@openai/codex-sdk"; +import type { AppConfig } from "../../config"; +import type { EventStore } from "../../persistence/event-store"; +import { readNativeGoalEvidence } from "./goal-evidence"; +import { formatImplementationPrompt } from "./prompts"; + +export async function runCodexWorker(input: { + prepared: { + attemptId: number; + worktreePath: string; + goal: string; + }; + signal: AbortSignal; + config: AppConfig["implementation"]; + store: EventStore; +}): Promise<{ codexThreadId: string; finalResponse: string }> { + const { prepared, config, store } = input; + const codex = new Codex({ + apiKey: config.model.apiKey, + config: { + model_provider: "implementer_local", + model_providers: { + implementer_local: { + name: "Configured local implementer", + base_url: config.model.baseUrl, + wire_api: "responses", + requires_openai_auth: false, + }, + }, + approvals_reviewer: "auto_review", + features: { goals: true }, + web_search: "disabled", + }, + }); + const thread = codex.startThread({ + model: config.model.modelId, + sandboxMode: "workspace-write", + approvalPolicy: "on-request", + workingDirectory: prepared.worktreePath, + networkAccessEnabled: false, + }); + const signal = AbortSignal.any([ + input.signal, + AbortSignal.timeout(config.timeoutMs), + ]); + const prompt = formatImplementationPrompt( + prepared.goal, + config.checkConfigPath, + ); + const { events } = await thread.runStreamed(prompt, { signal }); + let finalResponse = ""; + let threadId: string | null = null; + let turnFailure: string | null = null; + + for await (const event of events) { + store.appendImplementationEvent(prepared.attemptId, event); + if (event.type === "thread.started") { + threadId = event.thread_id; + store.setCodexThreadId(prepared.attemptId, threadId); + } + if (event.type === "item.completed" && event.item.type === "agent_message") { + finalResponse = event.item.text; + } + if (event.type === "turn.failed") turnFailure = event.error.message; + } + if (turnFailure) throw new Error(turnFailure); + if (!threadId) throw new Error("Codex did not return a thread ID."); + + const goalEvidence = readNativeGoalEvidence(threadId); + store.appendImplementationEvent(prepared.attemptId, { + type: "codex.goal.evidence", + ...goalEvidence, + }); + if (!goalEvidence.created) { + throw new Error("Codex finished without using the required native goal command."); + } + if (!goalEvidence.completed) { + throw new Error("Codex finished without completing its native goal."); + } + return { codexThreadId: threadId, finalResponse }; +} diff --git a/src/services/git/snapshot.ts b/src/services/git/snapshot.ts new file mode 100644 index 0000000..6d82c3c --- /dev/null +++ b/src/services/git/snapshot.ts @@ -0,0 +1,43 @@ +import { runCheckedProcess } from "../../shared/process"; + +export async function commitImplementationSnapshot(input: { + worktreePath: string; + subject: string; + authorName: string; + authorEmail: string; + signal: AbortSignal; +}): Promise { + const status = await runCheckedProcess("git", ["status", "--porcelain"], { + cwd: input.worktreePath, + timeoutMs: 30_000, + signal: input.signal, + }); + if (!status.output.trim()) { + throw new Error("Codex completed the goal without producing a repository change."); + } + await runCheckedProcess("git", ["add", "--all"], { + cwd: input.worktreePath, + timeoutMs: 30_000, + signal: input.signal, + }); + await runCheckedProcess( + "git", + [ + "-c", + `user.name=${input.authorName}`, + "-c", + `user.email=${input.authorEmail}`, + "commit", + "-m", + input.subject.slice(0, 200), + ], + { cwd: input.worktreePath, timeoutMs: 60_000, signal: input.signal }, + ); + return ( + await runCheckedProcess("git", ["rev-parse", "HEAD"], { + cwd: input.worktreePath, + timeoutMs: 30_000, + signal: input.signal, + }) + ).output.trim(); +} diff --git a/src/services/git/worktrees.ts b/src/services/git/worktrees.ts new file mode 100644 index 0000000..6faa9f6 --- /dev/null +++ b/src/services/git/worktrees.ts @@ -0,0 +1,39 @@ +import fs from "node:fs"; +import path from "node:path"; +import { runCheckedProcess } from "../../shared/process"; + +export async function createImplementationWorktree(input: { + repositoryPath: string; + worktreeRoot: string; + runId: string; + branch: string; + baseBranch: string; + signal: AbortSignal; +}): Promise<{ worktreePath: string; baseSha: string }> { + const baseSha = ( + await runCheckedProcess( + "git", + ["rev-parse", `refs/remotes/origin/${input.baseBranch}`], + { + cwd: input.repositoryPath, + timeoutMs: 30_000, + signal: input.signal, + }, + ) + ).output.trim(); + fs.mkdirSync(input.worktreeRoot, { recursive: true }); + const worktreePath = path.join(input.worktreeRoot, input.runId); + if (fs.existsSync(worktreePath)) { + throw new Error(`Worktree path already exists: ${worktreePath}`); + } + await runCheckedProcess( + "git", + ["worktree", "add", "-b", input.branch, worktreePath, baseSha], + { + cwd: input.repositoryPath, + timeoutMs: 60_000, + signal: input.signal, + }, + ); + return { worktreePath, baseSha }; +} diff --git a/src/github-comments.ts b/src/services/github/client.ts similarity index 58% rename from src/github-comments.ts rename to src/services/github/client.ts index e74cb90..e83ca3b 100644 --- a/src/github-comments.ts +++ b/src/services/github/client.ts @@ -1,6 +1,7 @@ import { createSign } from "node:crypto"; import fs from "node:fs"; -import type { AppConfig } from "./config"; +import type { AppConfig } from "../../config"; +import { runCheckedProcess } from "../../shared/process"; export interface PublishedGitHubComment { id: number; @@ -20,9 +21,35 @@ export interface GitHubCommentPublisher { ): Promise; } +export interface PublishedPullRequest { + number: number; + url: string; +} + +export interface GitHubPullRequestPublisher { + fetchBase( + repository: string, + repositoryPath: string, + baseBranch: string, + signal?: AbortSignal, + ): Promise; + pushBranch( + repository: string, + worktreePath: string, + branch: string, + signal?: AbortSignal, + ): Promise; + publishPullRequest( + repository: string, + options: { head: string; base: string; title: string; body: string }, + ): Promise; +} + type GitHubAppConfig = NonNullable; -export class GitHubAppCommentPublisher implements GitHubCommentPublisher { +export class GitHubAppCommentPublisher + implements GitHubCommentPublisher, GitHubPullRequestPublisher +{ private readonly privateKey: string; private cachedToken: { value: string; expiresAt: number } | null = null; @@ -67,6 +94,101 @@ export class GitHubAppCommentPublisher implements GitHubCommentPublisher { return { id: comment.id, url: comment.html_url }; } + async fetchBase( + repository: string, + repositoryPath: string, + baseBranch: string, + signal?: AbortSignal, + ): Promise { + await this.authenticatedGit( + repository, + repositoryPath, + [ + "fetch", + "--no-tags", + this.repositoryUrl(repository), + `+refs/heads/${baseBranch}:refs/remotes/origin/${baseBranch}`, + ], + signal, + ); + } + + async pushBranch( + repository: string, + worktreePath: string, + branch: string, + signal?: AbortSignal, + ): Promise { + await this.authenticatedGit( + repository, + worktreePath, + [ + "push", + "--set-upstream", + this.repositoryUrl(repository), + `HEAD:refs/heads/${branch}`, + ], + signal, + ); + } + + async publishPullRequest( + repository: string, + options: { head: string; base: string; title: string; body: string }, + ): Promise { + const { owner, name } = splitRepository(repository); + const existing = await this.request< + Array<{ number: number; html_url: string }> + >( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/pulls?state=open&head=${encodeURIComponent(`${owner}:${options.head}`)}&base=${encodeURIComponent(options.base)}`, + ); + if (existing[0]) { + return { number: existing[0].number, url: existing[0].html_url }; + } + const pullRequest = await this.request<{ + number: number; + html_url: string; + }>( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/pulls`, + { + method: "POST", + body: JSON.stringify({ + title: options.title, + body: options.body, + head: options.head, + base: options.base, + }), + }, + ); + return { number: pullRequest.number, url: pullRequest.html_url }; + } + + private repositoryUrl(repository: string): string { + const { owner, name } = splitRepository(repository); + return `${this.config.gitBaseUrl}/${encodeURIComponent(owner)}/${encodeURIComponent(name)}.git`; + } + + private async authenticatedGit( + repository: string, + cwd: string, + args: string[], + signal?: AbortSignal, + ): Promise { + const token = await this.installationToken(); + await runCheckedProcess("git", args, { + cwd, + timeoutMs: this.config.timeoutMs, + ...(signal ? { signal } : {}), + env: { + ...process.env, + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: `http.${this.repositoryUrl(repository)}.extraheader`, + GIT_CONFIG_VALUE_0: githubGitAuthorization(token), + GIT_TERMINAL_PROMPT: "0", + }, + }); + } + private async request(pathname: string, init: RequestInit = {}): Promise { const token = await this.installationToken(); const response = await this.fetchImplementation( @@ -143,6 +265,11 @@ export class GitHubAppCommentPublisher implements GitHubCommentPublisher { } } +export function githubGitAuthorization(token: string): string { + const credentials = Buffer.from(`x-access-token:${token}`).toString("base64"); + return `Authorization: Basic ${credentials}`; +} + function encodeJson(value: unknown): string { return Buffer.from(JSON.stringify(value)).toString("base64url"); } diff --git a/src/github-comment-outbox.ts b/src/services/github/comment-outbox.ts similarity index 95% rename from src/github-comment-outbox.ts rename to src/services/github/comment-outbox.ts index dd6af28..72da518 100644 --- a/src/github-comment-outbox.ts +++ b/src/services/github/comment-outbox.ts @@ -1,5 +1,5 @@ -import type { GitHubCommentPublisher } from "./github-comments"; -import type { EventStore } from "./store"; +import type { EventStore } from "../../persistence/event-store"; +import type { GitHubCommentPublisher } from "./client"; export class GitHubCommentOutboxWorker { private active = false; diff --git a/src/services/github/contracts.ts b/src/services/github/contracts.ts new file mode 100644 index 0000000..d03b3e3 --- /dev/null +++ b/src/services/github/contracts.ts @@ -0,0 +1,13 @@ +export interface NormalizedGitHubEvent { + deliveryId: string; + eventName: string; + action: string | null; + repository: string; + correlationKey: string; + issueNumber: number | null; + senderLogin: string | null; + isHumanComment: boolean; + commentBody: string | null; + requiresHuman: boolean; + payload: unknown; +} diff --git a/src/services/github/pull-request-body.ts b/src/services/github/pull-request-body.ts new file mode 100644 index 0000000..c355cdb --- /dev/null +++ b/src/services/github/pull-request-body.ts @@ -0,0 +1,26 @@ +import type { ImplementationCheckResult } from "../checks/contracts"; + +export function formatPullRequestBody(input: { + issueNumber: number | null; + summary: string; + checkResults: ImplementationCheckResult[]; + reviewSummary: string; +}): string { + const checks = input.checkResults + .map((check) => `- [x] ${check.name} (${check.durationMs} ms)`) + .join("\n"); + const closes = input.issueNumber === null ? "" : `\n\nCloses #${input.issueNumber}`; + return `## Implementation + +${input.summary || "Codex completed the native implementation goal."} + +## Deterministic checks + +${checks} + +## Review status + +${input.reviewSummary} + +The full accepted objective and acceptance criteria are preserved in the durable implementation record.${closes}`; +} diff --git a/src/github.ts b/src/services/github/webhook.ts similarity index 97% rename from src/github.ts rename to src/services/github/webhook.ts index 50f6d59..38d091d 100644 --- a/src/github.ts +++ b/src/services/github/webhook.ts @@ -1,5 +1,5 @@ import { createHmac, timingSafeEqual } from "node:crypto"; -import type { NormalizedGitHubEvent } from "./types"; +import type { NormalizedGitHubEvent } from "./contracts"; type GitHubPayload = Record; diff --git a/src/services/readiness/contracts.ts b/src/services/readiness/contracts.ts new file mode 100644 index 0000000..237d1f0 --- /dev/null +++ b/src/services/readiness/contracts.ts @@ -0,0 +1,28 @@ +export interface ReadinessInput { + controlRunId: string; + correlationKey: string; + repository: string; + issueNumber: number | null; + title: string; + body: string; + labels: string[]; + clarifications: string[]; +} + +export interface ReadinessEvaluation { + decision: TDecision; + modelId: string; + promptVersion: string; + traceId: string | null; + finishReason: string | null; + usage: unknown; +} + +export interface ReadinessEvaluator { + readonly modelId: string; + readonly promptVersion: string; + evaluate( + input: ReadinessInput, + options: { abortSignal: AbortSignal }, + ): Promise>; +} diff --git a/src/readiness.ts b/src/services/readiness/evaluator.ts similarity index 63% rename from src/readiness.ts rename to src/services/readiness/evaluator.ts index f0ed277..24d9acd 100644 --- a/src/readiness.ts +++ b/src/services/readiness/evaluator.ts @@ -1,54 +1,22 @@ import { Agent } from "@mastra/core/agent"; import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; import { z } from "zod"; -import type { ReadinessEvaluationResult, ReadinessInput } from "./types"; +import type { ReadinessEvaluator, ReadinessInput } from "./contracts"; export const READINESS_PROMPT_VERSION = "readiness-v1"; -export const readinessDecisionSchema = z - .object({ - ready: z.boolean(), - summary: z.string().min(1), - acceptanceCriteria: z.array(z.string().min(1)), - missingInformation: z.array(z.string().min(1)), - question: z.string().min(1).nullable(), - }) - .superRefine((decision, context) => { - if (decision.ready && decision.missingInformation.length > 0) { - context.addIssue({ - code: "custom", - message: "A ready issue cannot have missing information.", - }); - } - if (decision.ready && decision.question !== null) { - context.addIssue({ - code: "custom", - message: "A ready issue cannot ask a clarification question.", - }); - } - if (!decision.ready && decision.question === null) { - context.addIssue({ - code: "custom", - message: "An unready issue must ask a clarification question.", - }); - } - }); - -export interface ReadinessEvaluator { - readonly modelId: string; - readonly promptVersion: string; - evaluate( - input: ReadinessInput, - options: { abortSignal: AbortSignal }, - ): Promise; -} - -export function createReadinessEvaluator(config: { - baseUrl: string; - apiKey: string; - modelId: string; - timeoutMs: number; -}): { evaluator: ReadinessEvaluator; agent: Agent } { +export function createReadinessEvaluator( + config: { + baseUrl: string; + apiKey: string; + modelId: string; + timeoutMs: number; + }, + decisionSchema: z.ZodType, +): { + evaluator: ReadinessEvaluator; + agent: Agent; +} { const provider = createOpenAICompatible({ name: "local-llama", baseURL: config.baseUrl, @@ -79,12 +47,12 @@ export function createReadinessEvaluator(config: { const timeoutSignal = AbortSignal.timeout(config.timeoutMs); const result = await agent.generate(buildReadinessPrompt(input), { structuredOutput: { - schema: readinessDecisionSchema, + schema: decisionSchema, jsonPromptInjection: "auto", }, abortSignal: AbortSignal.any([options.abortSignal, timeoutSignal]), }); - const decision = readinessDecisionSchema.parse(result.object); + const decision = decisionSchema.parse(result.object); return { decision, modelId: config.modelId, diff --git a/src/shared/process.ts b/src/shared/process.ts new file mode 100644 index 0000000..d7a5a56 --- /dev/null +++ b/src/shared/process.ts @@ -0,0 +1,80 @@ +import { spawn } from "node:child_process"; + +export interface ProcessResult { + exitCode: number; + output: string; + durationMs: number; +} + +export async function runProcess( + command: string, + args: string[], + options: { + cwd: string; + timeoutMs: number; + signal?: AbortSignal; + env?: NodeJS.ProcessEnv; + maxOutputBytes?: number; + }, +): Promise { + const startedAt = Date.now(); + const timeoutSignal = AbortSignal.timeout(options.timeoutMs); + const signal = options.signal + ? AbortSignal.any([options.signal, timeoutSignal]) + : timeoutSignal; + const maxOutputBytes = options.maxOutputBytes ?? 200_000; + + return await new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env ?? process.env, + signal, + stdio: ["ignore", "pipe", "pipe"], + }); + const chunks: Buffer[] = []; + let outputBytes = 0; + let truncated = false; + + const capture = (chunk: Buffer | string) => { + if (outputBytes >= maxOutputBytes) { + truncated = true; + return; + } + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const remaining = maxOutputBytes - outputBytes; + chunks.push(buffer.subarray(0, remaining)); + outputBytes += Math.min(buffer.length, remaining); + if (buffer.length > remaining) truncated = true; + }; + child.stdout?.on("data", capture); + child.stderr?.on("data", capture); + child.once("error", reject); + child.once("close", (code, childSignal) => { + const suffix = truncated ? "\n[output truncated]" : ""; + const output = `${Buffer.concat(chunks).toString("utf8")}${suffix}`; + if (childSignal) { + reject(new Error(`${command} was terminated by ${childSignal}. ${output}`)); + return; + } + resolve({ + exitCode: code ?? 1, + output, + durationMs: Date.now() - startedAt, + }); + }); + }); +} + +export async function runCheckedProcess( + command: string, + args: string[], + options: Parameters[2], +): Promise { + const result = await runProcess(command, args, options); + if (result.exitCode !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed with exit code ${result.exitCode}.\n${result.output}`, + ); + } + return result; +} diff --git a/src/workflow.ts b/src/workflow.ts deleted file mode 100644 index c6c4a35..0000000 --- a/src/workflow.ts +++ /dev/null @@ -1,152 +0,0 @@ -import type { Agent } from "@mastra/core/agent"; -import { Mastra } from "@mastra/core/mastra"; -import { createStep, createWorkflow } from "@mastra/core/workflows"; -import { LibSQLStore } from "@mastra/libsql"; -import { PinoLogger } from "@mastra/loggers"; -import { MastraStorageExporter, Observability } from "@mastra/observability"; -import { z } from "zod"; -import { PRODUCTION_GRAPH_VERSION } from "./graph-definition"; -import { readinessDecisionSchema, type ReadinessEvaluator } from "./readiness"; -import type { EventStore } from "./store"; - -const workflowInputSchema = z.object({ - controlRunId: z.string(), - correlationKey: z.string(), - implementationMs: z.number().int().nonnegative(), - queuedEventsSeen: z.number().int().nonnegative().default(0), -}); - -const readyDataSchema = workflowInputSchema.extend({ - readiness: readinessDecisionSchema, - readinessEvaluationId: z.number().int().positive(), -}); - -const resumeSchema = z.object({ answer: z.string().min(1) }); -const readinessSuspendSchema = z.object({ - question: z.string(), - missingInformation: z.array(z.string()), - decision: readinessDecisionSchema, - evaluationId: z.number().int().positive(), -}); - -export function createImplementationMastra(params: { - databaseUrl: string; - eventStore: EventStore; - readinessEvaluator: ReadinessEvaluator; - readinessAgent?: Agent; -}) { - const readiness = createStep({ - id: "readiness", - description: "Verify that the work item is sufficiently specified.", - inputSchema: workflowInputSchema, - outputSchema: readyDataSchema, - resumeSchema, - suspendSchema: readinessSuspendSchema, - retries: 1, - execute: async ({ inputData, resumeData, suspend, abortSignal }) => { - const readinessInput = params.eventStore.getReadinessInput( - inputData.controlRunId, - ); - if ( - resumeData?.answer && - !readinessInput.clarifications.includes(resumeData.answer) - ) { - readinessInput.clarifications.push(resumeData.answer); - } - const evaluationId = params.eventStore.startReadinessEvaluation( - inputData.controlRunId, - readinessInput, - params.readinessEvaluator.modelId, - params.readinessEvaluator.promptVersion, - PRODUCTION_GRAPH_VERSION, - ); - - try { - const evaluation = await params.readinessEvaluator.evaluate( - readinessInput, - { abortSignal }, - ); - params.eventStore.completeReadinessEvaluation(evaluationId, evaluation); - if (!evaluation.decision.ready) { - return suspend({ - question: - evaluation.decision.question ?? - "What information is required before implementation can begin?", - missingInformation: evaluation.decision.missingInformation, - decision: evaluation.decision, - evaluationId, - }); - } - return { - ...inputData, - readiness: evaluation.decision, - readinessEvaluationId: evaluationId, - }; - } catch (error) { - params.eventStore.failReadinessEvaluation(evaluationId, error); - throw error; - } - }, - }); - - const implementation = createStep({ - id: "implementation", - inputSchema: readyDataSchema, - outputSchema: readyDataSchema, - execute: async ({ inputData, abortSignal }) => { - await abortableDelay(inputData.implementationMs, abortSignal); - const queuedEventsSeen = params.eventStore.countQueuedEvents( - inputData.controlRunId, - ); - return { ...inputData, queuedEventsSeen }; - }, - }); - - const workflow = createWorkflow({ - id: "github-implementation", - inputSchema: workflowInputSchema, - outputSchema: readyDataSchema, - }) - .then(readiness) - .then(implementation) - .commit(); - - const storage = new LibSQLStore({ id: "loop-storage", url: params.databaseUrl }); - const mastra = new Mastra({ - storage, - ...(params.readinessAgent - ? { agents: { readiness: params.readinessAgent } } - : {}), - workflows: { implementation: workflow }, - logger: new PinoLogger({ name: "Implementer", level: "info" }), - observability: new Observability({ - configs: { - default: { - serviceName: "implementer", - exporters: [new MastraStorageExporter()], - logging: { enabled: true, level: "info" }, - }, - }, - }), - }); - - return { mastra, workflow, storage }; -} - -function abortableDelay(milliseconds: number, signal: AbortSignal): Promise { - return new Promise((resolve, reject) => { - if (signal.aborted) { - reject(signal.reason); - return; - } - const timer = setTimeout(resolve, milliseconds); - signal.addEventListener( - "abort", - () => { - clearTimeout(timer); - reject(signal.reason); - }, - { once: true }, - ); - }); -} diff --git a/src/graph-definition.ts b/src/workflows/definitions.ts similarity index 84% rename from src/graph-definition.ts rename to src/workflows/definitions.ts index da12e30..c519d5f 100644 --- a/src/graph-definition.ts +++ b/src/workflows/definitions.ts @@ -1,5 +1,6 @@ -/** Machine-readable contract for the graph we are building. */ +/** Machine-readable contract for the graphs we are building. */ export const PRODUCTION_GRAPH_VERSION = "production-v1"; +export const DEMO_GRAPH_VERSION = "demo-v1"; export type NodeStatus = "implemented" | "partial" | "planned"; @@ -41,13 +42,20 @@ export const productionGraph: readonly GraphNodeDefinition[] = [ { id: "github-ingest", responsibility: "Persist, deduplicate, correlate, and acknowledge GitHub events.", kind: "deterministic", status: "implemented" }, { id: "readiness-and-decomposition", responsibility: "Verify sufficient context and decide whether the issue should be split into child issues.", kind: "agent", status: "partial" }, { id: "human-clarification", responsibility: "Suspend and resume when missing information or a product decision requires a human.", kind: "human-gate", status: "partial" }, - { id: "codex-goal-implementation", responsibility: "Implement one accepted issue in an isolated worktree against explicit acceptance criteria.", kind: "agent", status: "planned" }, - { id: "deterministic-checks", responsibility: "Run repository-owned tests, builds, linting, type checks, and policy checks.", kind: "deterministic", status: "planned" }, - { id: "specialist-reviewers", responsibility: "Run the independent reviewer set in parallel against one fixed implementation snapshot.", kind: "parallel", status: "planned" }, + { id: "codex-goal-implementation", responsibility: "Implement one accepted issue in an isolated worktree against explicit acceptance criteria.", kind: "agent", status: "implemented" }, + { id: "deterministic-checks", responsibility: "Run repository-owned tests, builds, linting, type checks, and policy checks.", kind: "deterministic", status: "implemented" }, + { id: "specialist-reviewers", responsibility: "Run the independent reviewer set in parallel against one fixed implementation snapshot.", kind: "parallel", status: "partial" }, { id: "review-manager", responsibility: "Consolidate evidence, resolve compatible findings, route repairs, and expose true conflicts.", kind: "agent", status: "planned" }, { id: "human-conflict-decision", responsibility: "Choose between irreconcilable recommendations using their evidence and trade-offs.", kind: "human-gate", status: "planned" }, - { id: "pull-request", responsibility: "Publish an approved, verified change and its evidence exactly once.", kind: "side-effect", status: "planned" }, - { id: "recorder", responsibility: "Append reconstructable run evidence without modifying the active graph.", kind: "deterministic", status: "planned" }, + { id: "pull-request", responsibility: "Publish an approved, verified change and its evidence exactly once.", kind: "side-effect", status: "partial" }, + { id: "recorder", responsibility: "Append reconstructable run evidence without modifying the active graph.", kind: "deterministic", status: "partial" }, +] as const; + +export const demoImplementationGraph: readonly GraphNodeDefinition[] = [ + { id: "readiness", responsibility: "Verify that the issue is sufficiently specified before the demo writes code.", kind: "agent", status: "implemented" }, + { id: "prepare-worktree", responsibility: "Create one isolated branch and worktree from the configured remote base.", kind: "deterministic", status: "implemented" }, + { id: "demo-codex-implementation", responsibility: "Run one write-capable Codex turn without creating or validating a persistent goal.", kind: "agent", status: "implemented" }, + { id: "demo-pull-request", responsibility: "Commit, push, and create an explicitly low-assurance pull request without deterministic checks or independent review.", kind: "side-effect", status: "implemented" }, ] as const; export const selfImprovementGraph: readonly GraphNodeDefinition[] = [ diff --git a/src/workflows/demo-implementation/create-step-implementations.ts b/src/workflows/demo-implementation/create-step-implementations.ts new file mode 100644 index 0000000..4b8f47f --- /dev/null +++ b/src/workflows/demo-implementation/create-step-implementations.ts @@ -0,0 +1,45 @@ +import type { AppConfig } from "../../config"; +import type { EventStore } from "../../persistence/event-store"; +import type { GitHubPullRequestPublisher } from "../../services/github/client"; +import type { ReadinessEvaluator } from "../../services/readiness/contracts"; +import { DEMO_IMPLEMENTATION_PROMPT_VERSION } from "../../services/codex/prompts"; +import { DEMO_GRAPH_VERSION } from "../definitions"; +import { PrepareWorktreeImplementation } from "../implementation/steps/prepare-worktree/prepare-worktree.implementation"; +import type { ReadinessDecision } from "../implementation/steps/readiness/readiness.definition"; +import { ReadinessImplementation } from "../implementation/steps/readiness/readiness.implementation"; +import { DemoCodexImplementation } from "./steps/codex-implementation/codex-implementation.implementation"; +import { DemoPublishPullRequestImplementation } from "./steps/publish-pull-request/publish-pull-request.implementation"; + +export interface DemoImplementationWorkflowImplementations { + readiness: Pick; + prepareWorktree: Pick; + codexImplementation: Pick; + publishPullRequest: Pick; +} + +export function createDemoImplementationWorkflowImplementations(input: { + config: AppConfig["implementation"]; + store: EventStore; + readinessEvaluator: ReadinessEvaluator; + publisher: GitHubPullRequestPublisher | null; +}): DemoImplementationWorkflowImplementations { + return { + readiness: new ReadinessImplementation( + input.store, + input.readinessEvaluator, + DEMO_GRAPH_VERSION, + ), + prepareWorktree: new PrepareWorktreeImplementation( + input.config, + input.store, + input.publisher, + DEMO_IMPLEMENTATION_PROMPT_VERSION, + ), + codexImplementation: new DemoCodexImplementation(input.config, input.store), + publishPullRequest: new DemoPublishPullRequestImplementation( + input.config, + input.store, + input.publisher, + ), + }; +} diff --git a/src/workflows/demo-implementation/steps/codex-implementation/codex-implementation.definition.ts b/src/workflows/demo-implementation/steps/codex-implementation/codex-implementation.definition.ts new file mode 100644 index 0000000..fde30bc --- /dev/null +++ b/src/workflows/demo-implementation/steps/codex-implementation/codex-implementation.definition.ts @@ -0,0 +1,29 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { prepareWorktreeOutputSchema } from "../../../implementation/steps/prepare-worktree/prepare-worktree.definition"; +import type { DemoCodexImplementation } from "./codex-implementation.implementation"; + +export const demoCodexImplementationOutputSchema = + prepareWorktreeOutputSchema.extend({ + codexThreadId: z.string(), + finalResponse: z.string(), + }); + +export type DemoCodexImplementationOutput = z.infer< + typeof demoCodexImplementationOutputSchema +>; + +export function createDemoCodexImplementationStep( + implementation: Pick, +) { + return createStep({ + id: "demo-codex-implementation", + description: "Implement the ready issue with Codex without a persistent goal.", + inputSchema: prepareWorktreeOutputSchema, + outputSchema: demoCodexImplementationOutputSchema, + execute: async ({ inputData, abortSignal }) => ({ + ...inputData, + ...(await implementation.execute(inputData, abortSignal)), + }), + }); +} diff --git a/src/workflows/demo-implementation/steps/codex-implementation/codex-implementation.implementation.ts b/src/workflows/demo-implementation/steps/codex-implementation/codex-implementation.implementation.ts new file mode 100644 index 0000000..8ee3968 --- /dev/null +++ b/src/workflows/demo-implementation/steps/codex-implementation/codex-implementation.implementation.ts @@ -0,0 +1,35 @@ +import type { AppConfig } from "../../../../config"; +import type { EventStore } from "../../../../persistence/event-store"; +import { runDemoCodexWorker } from "../../../../services/codex/demo-worker"; +import type { PrepareWorktreeOutput } from "../../../implementation/steps/prepare-worktree/prepare-worktree.definition"; +import type { DemoCodexImplementationOutput } from "./codex-implementation.definition"; + +export class DemoCodexImplementation { + constructor( + private readonly config: AppConfig["implementation"], + private readonly store: EventStore, + ) {} + + async execute( + input: PrepareWorktreeOutput, + signal: AbortSignal, + ): Promise { + this.store.setImplementationStage(input.attemptId, "implementing"); + try { + const result = await runDemoCodexWorker({ + prepared: input, + signal, + config: this.config, + store: this.store, + }); + this.store.completeCodexImplementation( + input.attemptId, + result.finalResponse, + ); + return { ...input, ...result }; + } catch (error) { + this.store.failImplementationAttempt(input.attemptId, error); + throw error; + } + } +} diff --git a/src/workflows/demo-implementation/steps/publish-pull-request/publish-pull-request.definition.ts b/src/workflows/demo-implementation/steps/publish-pull-request/publish-pull-request.definition.ts new file mode 100644 index 0000000..72f20f4 --- /dev/null +++ b/src/workflows/demo-implementation/steps/publish-pull-request/publish-pull-request.definition.ts @@ -0,0 +1,28 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { demoCodexImplementationOutputSchema } from "../codex-implementation/codex-implementation.definition"; +import type { DemoPublishPullRequestImplementation } from "./publish-pull-request.implementation"; + +export const demoPublishPullRequestOutputSchema = + demoCodexImplementationOutputSchema.extend({ + commitSha: z.string(), + pullRequestUrl: z.string().url(), + }); + +export type DemoPublishPullRequestOutput = z.infer< + typeof demoPublishPullRequestOutputSchema +>; + +export function createDemoPublishPullRequestStep( + implementation: Pick, +) { + return createStep({ + id: "demo-publish-pull-request", + description: + "Commit and publish the demo result without deterministic checks or review.", + inputSchema: demoCodexImplementationOutputSchema, + outputSchema: demoPublishPullRequestOutputSchema, + execute: async ({ inputData, abortSignal }) => + implementation.execute(inputData, abortSignal), + }); +} diff --git a/src/workflows/demo-implementation/steps/publish-pull-request/publish-pull-request.implementation.ts b/src/workflows/demo-implementation/steps/publish-pull-request/publish-pull-request.implementation.ts new file mode 100644 index 0000000..f309229 --- /dev/null +++ b/src/workflows/demo-implementation/steps/publish-pull-request/publish-pull-request.implementation.ts @@ -0,0 +1,80 @@ +import type { AppConfig } from "../../../../config"; +import type { EventStore } from "../../../../persistence/event-store"; +import { commitImplementationSnapshot } from "../../../../services/git/snapshot"; +import type { GitHubPullRequestPublisher } from "../../../../services/github/client"; +import type { DemoCodexImplementationOutput } from "../codex-implementation/codex-implementation.definition"; +import type { DemoPublishPullRequestOutput } from "./publish-pull-request.definition"; + +export class DemoPublishPullRequestImplementation { + constructor( + private readonly config: AppConfig["implementation"], + private readonly store: EventStore, + private readonly publisher: GitHubPullRequestPublisher | null, + ) {} + + async execute( + input: DemoCodexImplementationOutput, + signal: AbortSignal, + ): Promise { + try { + if (!this.publisher) { + throw new Error("GitHub pull-request publishing is unavailable."); + } + const attempt = this.store.getImplementationAttempt(input.attemptId); + if (!attempt) { + throw new Error(`Implementation attempt ${input.attemptId} is missing.`); + } + const readinessInput = this.store.getReadinessInput(attempt.runId); + this.store.setImplementationStage(input.attemptId, "snapshotting"); + const commitSha = await commitImplementationSnapshot({ + worktreePath: input.worktreePath, + subject: `Demo implement #${readinessInput.issueNumber ?? "event"}: ${readinessInput.title}`, + authorName: this.config.gitAuthorName, + authorEmail: this.config.gitAuthorEmail, + signal, + }); + this.store.recordImplementationSnapshot(input.attemptId, commitSha); + + this.store.setImplementationStage(input.attemptId, "publishing"); + await this.publisher.pushBranch( + this.config.repository, + input.worktreePath, + input.branch, + signal, + ); + const pullRequest = await this.publisher.publishPullRequest( + this.config.repository, + { + head: input.branch, + base: this.config.baseBranch, + title: + readinessInput.title || + `Demo implement issue #${readinessInput.issueNumber}`, + body: formatDemoPullRequestBody({ + issueNumber: readinessInput.issueNumber, + summary: input.finalResponse, + }), + }, + ); + this.store.completeImplementationAttempt(input.attemptId, pullRequest.url); + return { ...input, commitSha, pullRequestUrl: pullRequest.url }; + } catch (error) { + this.store.failImplementationAttempt(input.attemptId, error); + throw error; + } + } +} + +function formatDemoPullRequestBody(input: { + issueNumber: number | null; + summary: string; +}): string { + const closes = input.issueNumber === null ? "" : `\n\nCloses #${input.issueNumber}`; + return `## Demo implementation + +${input.summary || "Codex completed the requested implementation."} + +## Assurance boundary + +This pull request was created by the reduced demo workflow after readiness and an isolated Codex implementation. It did not run the externally managed deterministic checks, persistent goal validation, or independent review used by the production workflow.${closes}`; +} diff --git a/src/workflows/demo-implementation/workflow.ts b/src/workflows/demo-implementation/workflow.ts new file mode 100644 index 0000000..2e879b6 --- /dev/null +++ b/src/workflows/demo-implementation/workflow.ts @@ -0,0 +1,24 @@ +import { createWorkflow } from "@mastra/core/workflows"; +import type { DemoImplementationWorkflowImplementations } from "./create-step-implementations"; +import { createReadinessStep, workflowInputSchema } from "../implementation/steps/readiness/readiness.definition"; +import { createPrepareWorktreeStep } from "../implementation/steps/prepare-worktree/prepare-worktree.definition"; +import { createDemoCodexImplementationStep } from "./steps/codex-implementation/codex-implementation.definition"; +import { + createDemoPublishPullRequestStep, + demoPublishPullRequestOutputSchema, +} from "./steps/publish-pull-request/publish-pull-request.definition"; + +export function createDemoImplementationWorkflow( + implementations: DemoImplementationWorkflowImplementations, +) { + return createWorkflow({ + id: "github-demo-implementation", + inputSchema: workflowInputSchema, + outputSchema: demoPublishPullRequestOutputSchema, + }) + .then(createReadinessStep(implementations.readiness)) + .then(createPrepareWorktreeStep(implementations.prepareWorktree)) + .then(createDemoCodexImplementationStep(implementations.codexImplementation)) + .then(createDemoPublishPullRequestStep(implementations.publishPullRequest)) + .commit(); +} diff --git a/src/workflows/implementation/create-step-implementations.ts b/src/workflows/implementation/create-step-implementations.ts new file mode 100644 index 0000000..6bde954 --- /dev/null +++ b/src/workflows/implementation/create-step-implementations.ts @@ -0,0 +1,50 @@ +import type { AppConfig } from "../../config"; +import type { EventStore } from "../../persistence/event-store"; +import type { GitHubPullRequestPublisher } from "../../services/github/client"; +import type { ReadinessEvaluator } from "../../services/readiness/contracts"; +import { CodexImplementation } from "./steps/codex-implementation/codex-implementation.implementation"; +import { DeterministicChecksImplementation } from "./steps/deterministic-checks/deterministic-checks.implementation"; +import { PrepareWorktreeImplementation } from "./steps/prepare-worktree/prepare-worktree.implementation"; +import { PublishPullRequestImplementation } from "./steps/publish-pull-request/publish-pull-request.implementation"; +import type { ReadinessDecision } from "./steps/readiness/readiness.definition"; +import { ReadinessImplementation } from "./steps/readiness/readiness.implementation"; +import { ReviewerImplementation } from "./steps/reviewer/reviewer.implementation"; +import { SnapshotImplementation } from "./steps/snapshot/snapshot.implementation"; + +export interface ImplementationWorkflowImplementations { + readiness: Pick; + prepareWorktree: Pick; + codexImplementation: Pick; + deterministicChecks: Pick; + snapshot: Pick; + reviewer: Pick; + publishPullRequest: Pick; +} + +export function createImplementationWorkflowImplementations(input: { + config: AppConfig["implementation"]; + store: EventStore; + readinessEvaluator: ReadinessEvaluator; + publisher: GitHubPullRequestPublisher | null; +}): ImplementationWorkflowImplementations { + return { + readiness: new ReadinessImplementation(input.store, input.readinessEvaluator), + prepareWorktree: new PrepareWorktreeImplementation( + input.config, + input.store, + input.publisher, + ), + codexImplementation: new CodexImplementation(input.config, input.store), + deterministicChecks: new DeterministicChecksImplementation( + input.config, + input.store, + ), + snapshot: new SnapshotImplementation(input.config, input.store), + reviewer: new ReviewerImplementation(input.store), + publishPullRequest: new PublishPullRequestImplementation( + input.config, + input.store, + input.publisher, + ), + }; +} diff --git a/src/workflows/implementation/dependencies.ts b/src/workflows/implementation/dependencies.ts new file mode 100644 index 0000000..7b2b16f --- /dev/null +++ b/src/workflows/implementation/dependencies.ts @@ -0,0 +1,10 @@ +import type { Agent } from "@mastra/core/agent"; +import type { DemoImplementationWorkflowImplementations } from "../demo-implementation/create-step-implementations"; +import type { ImplementationWorkflowImplementations } from "./create-step-implementations"; + +export interface ImplementationWorkflowDependencies { + databaseUrl: string; + implementations: ImplementationWorkflowImplementations; + demoImplementations: DemoImplementationWorkflowImplementations; + readinessAgent?: Agent; +} diff --git a/src/workflows/implementation/index.ts b/src/workflows/implementation/index.ts new file mode 100644 index 0000000..07cdb40 --- /dev/null +++ b/src/workflows/implementation/index.ts @@ -0,0 +1 @@ +export { createImplementationMastra } from "./workflow"; diff --git a/src/workflows/implementation/steps/codex-implementation/codex-implementation.definition.ts b/src/workflows/implementation/steps/codex-implementation/codex-implementation.definition.ts new file mode 100644 index 0000000..e7cc80f --- /dev/null +++ b/src/workflows/implementation/steps/codex-implementation/codex-implementation.definition.ts @@ -0,0 +1,28 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { prepareWorktreeOutputSchema } from "../prepare-worktree/prepare-worktree.definition"; +import type { CodexImplementation } from "./codex-implementation.implementation"; + +export const codexImplementationOutputSchema = + prepareWorktreeOutputSchema.extend({ + codexThreadId: z.string(), + finalResponse: z.string(), + }); + +export type CodexImplementationOutput = z.infer< + typeof codexImplementationOutputSchema +>; + +export function createCodexImplementationStep( + implementation: Pick, +) { + return createStep({ + id: "codex-goal-implementation", + inputSchema: prepareWorktreeOutputSchema, + outputSchema: codexImplementationOutputSchema, + execute: async ({ inputData, abortSignal }) => ({ + ...inputData, + ...(await implementation.execute(inputData, abortSignal)), + }), + }); +} diff --git a/src/workflows/implementation/steps/codex-implementation/codex-implementation.implementation.ts b/src/workflows/implementation/steps/codex-implementation/codex-implementation.implementation.ts new file mode 100644 index 0000000..dcaa0c7 --- /dev/null +++ b/src/workflows/implementation/steps/codex-implementation/codex-implementation.implementation.ts @@ -0,0 +1,35 @@ +import type { AppConfig } from "../../../../config"; +import type { EventStore } from "../../../../persistence/event-store"; +import { runCodexWorker } from "../../../../services/codex/worker"; +import type { PrepareWorktreeOutput } from "../prepare-worktree/prepare-worktree.definition"; +import type { CodexImplementationOutput } from "./codex-implementation.definition"; + +export class CodexImplementation { + constructor( + private readonly config: AppConfig["implementation"], + private readonly store: EventStore, + ) {} + + async execute( + input: PrepareWorktreeOutput, + signal: AbortSignal, + ): Promise { + this.store.setImplementationStage(input.attemptId, "implementing"); + try { + const result = await runCodexWorker({ + prepared: input, + signal, + config: this.config, + store: this.store, + }); + this.store.completeCodexImplementation( + input.attemptId, + result.finalResponse, + ); + return { ...input, ...result }; + } catch (error) { + this.store.failImplementationAttempt(input.attemptId, error); + throw error; + } + } +} diff --git a/src/workflows/implementation/steps/deterministic-checks/deterministic-checks.definition.ts b/src/workflows/implementation/steps/deterministic-checks/deterministic-checks.definition.ts new file mode 100644 index 0000000..5ba2cc5 --- /dev/null +++ b/src/workflows/implementation/steps/deterministic-checks/deterministic-checks.definition.ts @@ -0,0 +1,35 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { codexImplementationOutputSchema } from "../codex-implementation/codex-implementation.definition"; +import type { DeterministicChecksImplementation } from "./deterministic-checks.implementation"; + +const checkResultSchema = z.object({ + name: z.string(), + command: z.array(z.string()), + exitCode: z.number().int(), + output: z.string(), + durationMs: z.number().nonnegative(), +}); + +export const deterministicChecksOutputSchema = + codexImplementationOutputSchema.extend({ + checkResults: z.array(checkResultSchema), + }); + +export type DeterministicChecksOutput = z.infer< + typeof deterministicChecksOutputSchema +>; + +export function createDeterministicChecksStep( + implementation: Pick, +) { + return createStep({ + id: "deterministic-checks", + inputSchema: codexImplementationOutputSchema, + outputSchema: deterministicChecksOutputSchema, + execute: async ({ inputData, abortSignal }) => ({ + ...inputData, + ...(await implementation.execute(inputData, abortSignal)), + }), + }); +} diff --git a/src/workflows/implementation/steps/deterministic-checks/deterministic-checks.implementation.ts b/src/workflows/implementation/steps/deterministic-checks/deterministic-checks.implementation.ts new file mode 100644 index 0000000..6db2ad9 --- /dev/null +++ b/src/workflows/implementation/steps/deterministic-checks/deterministic-checks.implementation.ts @@ -0,0 +1,47 @@ +import type { AppConfig } from "../../../../config"; +import type { EventStore } from "../../../../persistence/event-store"; +import { runDeterministicChecks } from "../../../../services/checks/runner"; +import { runProcess } from "../../../../shared/process"; +import type { CodexImplementationOutput } from "../codex-implementation/codex-implementation.definition"; +import type { DeterministicChecksOutput } from "./deterministic-checks.definition"; + +export class DeterministicChecksImplementation { + constructor( + private readonly config: AppConfig["implementation"], + private readonly store: EventStore, + ) {} + + async execute( + input: CodexImplementationOutput, + signal: AbortSignal, + ): Promise { + this.store.setImplementationStage(input.attemptId, "checking"); + try { + const attempt = this.store.getImplementationAttempt(input.attemptId); + if (!attempt) { + throw new Error(`Implementation attempt ${input.attemptId} is missing.`); + } + const configDiff = await runProcess( + "git", + ["diff", "--quiet", input.baseSha, "--", this.config.checkConfigPath], + { cwd: input.worktreePath, timeoutMs: 30_000, signal }, + ); + if (configDiff.exitCode !== 0) { + throw new Error( + `${this.config.checkConfigPath} changed during implementation; checks must remain externally defined.`, + ); + } + const checkResults = await runDeterministicChecks({ + checks: attempt.checks, + attemptId: input.attemptId, + worktreePath: input.worktreePath, + signal, + store: this.store, + }); + return { ...input, checkResults }; + } catch (error) { + this.store.failImplementationAttempt(input.attemptId, error); + throw error; + } + } +} diff --git a/src/workflows/implementation/steps/prepare-worktree/prepare-worktree.definition.ts b/src/workflows/implementation/steps/prepare-worktree/prepare-worktree.definition.ts new file mode 100644 index 0000000..25e6aa3 --- /dev/null +++ b/src/workflows/implementation/steps/prepare-worktree/prepare-worktree.definition.ts @@ -0,0 +1,40 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { readinessOutputSchema } from "../readiness/readiness.definition"; +import type { PrepareWorktreeImplementation } from "./prepare-worktree.implementation"; + +const preparedWorktreeSchema = z.object({ + attemptId: z.number().int().positive(), + worktreePath: z.string(), + branch: z.string(), + baseSha: z.string(), + goal: z.string(), +}); + +export type PreparedWorktree = z.infer; + +export const prepareWorktreeOutputSchema = readinessOutputSchema.extend( + preparedWorktreeSchema.shape, +); + +export type PrepareWorktreeOutput = z.infer< + typeof prepareWorktreeOutputSchema +>; + +export function createPrepareWorktreeStep( + implementation: Pick, +) { + return createStep({ + id: "prepare-worktree", + inputSchema: readinessOutputSchema, + outputSchema: prepareWorktreeOutputSchema, + execute: async ({ inputData, abortSignal }) => { + const prepared = await implementation.execute({ + controlRunId: inputData.controlRunId, + readiness: inputData.readiness, + signal: abortSignal, + }); + return { ...inputData, ...prepared }; + }, + }); +} diff --git a/src/workflows/implementation/steps/prepare-worktree/prepare-worktree.implementation.ts b/src/workflows/implementation/steps/prepare-worktree/prepare-worktree.implementation.ts new file mode 100644 index 0000000..ccc1a5e --- /dev/null +++ b/src/workflows/implementation/steps/prepare-worktree/prepare-worktree.implementation.ts @@ -0,0 +1,102 @@ +import path from "node:path"; +import type { AppConfig } from "../../../../config"; +import type { EventStore } from "../../../../persistence/event-store"; +import { loadRepositoryCheckConfiguration } from "../../../../services/checks/configuration"; +import { runSetupCommands } from "../../../../services/checks/runner"; +import { + formatGoal, + IMPLEMENTATION_PROMPT_VERSION, +} from "../../../../services/codex/prompts"; +import { createImplementationWorktree } from "../../../../services/git/worktrees"; +import type { GitHubPullRequestPublisher } from "../../../../services/github/client"; +import type { ReadinessDecision } from "../readiness/readiness.definition"; +import type { PreparedWorktree } from "./prepare-worktree.definition"; + +export class PrepareWorktreeImplementation { + constructor( + private readonly config: AppConfig["implementation"], + private readonly store: EventStore, + private readonly publisher: GitHubPullRequestPublisher | null, + private readonly promptVersion = IMPLEMENTATION_PROMPT_VERSION, + ) {} + + async execute(input: { + controlRunId: string; + readiness: ReadinessDecision; + signal: AbortSignal; + }): Promise { + if (!this.publisher) { + throw new Error( + "The GitHub App is required before implementation can create and publish a pull request.", + ); + } + const readinessInput = this.store.getReadinessInput(input.controlRunId); + if (readinessInput.repository !== this.config.repository) { + throw new Error( + `Run targets ${readinessInput.repository}, but the implementer is configured for ${this.config.repository}.`, + ); + } + const suffix = input.controlRunId.replaceAll("-", "").slice(0, 10); + const issuePart = readinessInput.issueNumber ?? "event"; + const branch = `codex/issue-${issuePart}-${suffix}`; + const worktreePath = path.join(this.config.worktreeRoot, input.controlRunId); + const goal = formatGoal({ + title: readinessInput.title, + body: readinessInput.body, + clarifications: readinessInput.clarifications, + acceptanceCriteria: input.readiness.acceptanceCriteria, + }); + const attemptId = this.store.startImplementationAttempt({ + runId: input.controlRunId, + repositoryPath: this.config.repositoryPath, + worktreePath, + branch, + goal, + modelBaseUrl: this.config.model.baseUrl, + modelId: this.config.model.modelId, + promptVersion: this.promptVersion, + }); + + try { + await this.publisher.fetchBase( + this.config.repository, + this.config.repositoryPath, + this.config.baseBranch, + input.signal, + ); + const worktree = await createImplementationWorktree({ + repositoryPath: this.config.repositoryPath, + worktreeRoot: this.config.worktreeRoot, + runId: input.controlRunId, + branch, + baseBranch: this.config.baseBranch, + signal: input.signal, + }); + const repositoryConfig = loadRepositoryCheckConfiguration( + path.join(worktree.worktreePath, this.config.checkConfigPath), + ); + this.store.configureImplementationAttempt( + attemptId, + worktree.baseSha, + repositoryConfig.checks, + ); + await runSetupCommands({ + commands: repositoryConfig.setup, + attemptId, + worktreePath: worktree.worktreePath, + signal: input.signal, + store: this.store, + }); + return { + attemptId, + worktreePath: worktree.worktreePath, + branch, + baseSha: worktree.baseSha, + goal, + }; + } catch (error) { + this.store.failImplementationAttempt(attemptId, error); + throw error; + } + } +} diff --git a/src/workflows/implementation/steps/publish-pull-request/publish-pull-request.definition.ts b/src/workflows/implementation/steps/publish-pull-request/publish-pull-request.definition.ts new file mode 100644 index 0000000..1388717 --- /dev/null +++ b/src/workflows/implementation/steps/publish-pull-request/publish-pull-request.definition.ts @@ -0,0 +1,26 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { reviewerOutputSchema } from "../reviewer/reviewer.definition"; +import type { PublishPullRequestImplementation } from "./publish-pull-request.implementation"; + +export const publishPullRequestOutputSchema = reviewerOutputSchema.extend({ + pullRequestUrl: z.url(), +}); + +export type PublishPullRequestOutput = z.infer< + typeof publishPullRequestOutputSchema +>; + +export function createPublishPullRequestStep( + implementation: Pick, +) { + return createStep({ + id: "pull-request", + inputSchema: reviewerOutputSchema, + outputSchema: publishPullRequestOutputSchema, + execute: async ({ inputData, abortSignal }) => { + const published = await implementation.execute(inputData, abortSignal); + return { ...inputData, ...published }; + }, + }); +} diff --git a/src/workflows/implementation/steps/publish-pull-request/publish-pull-request.implementation.ts b/src/workflows/implementation/steps/publish-pull-request/publish-pull-request.implementation.ts new file mode 100644 index 0000000..8c2564a --- /dev/null +++ b/src/workflows/implementation/steps/publish-pull-request/publish-pull-request.implementation.ts @@ -0,0 +1,62 @@ +import type { AppConfig } from "../../../../config"; +import type { EventStore } from "../../../../persistence/event-store"; +import type { GitHubPullRequestPublisher } from "../../../../services/github/client"; +import { formatPullRequestBody } from "../../../../services/github/pull-request-body"; +import type { ReviewerOutput } from "../reviewer/reviewer.definition"; +import type { PublishPullRequestOutput } from "./publish-pull-request.definition"; + +export class PublishPullRequestImplementation { + constructor( + private readonly config: AppConfig["implementation"], + private readonly store: EventStore, + private readonly publisher: GitHubPullRequestPublisher | null, + ) {} + + async execute( + input: ReviewerOutput, + signal: AbortSignal, + ): Promise { + this.store.setImplementationStage(input.attemptId, "publishing"); + try { + if (!this.publisher) { + throw new Error("GitHub pull-request publishing is unavailable."); + } + const attempt = this.store.getImplementationAttempt(input.attemptId); + if (!attempt) { + throw new Error(`Implementation attempt ${input.attemptId} is missing.`); + } + const readinessInput = this.store.getReadinessInput(attempt.runId); + await this.publisher.pushBranch( + this.config.repository, + input.worktreePath, + input.branch, + signal, + ); + const pullRequest = await this.publisher.publishPullRequest( + this.config.repository, + { + head: input.branch, + base: this.config.baseBranch, + title: + readinessInput.title || + `Implement issue #${readinessInput.issueNumber}`, + body: formatPullRequestBody({ + issueNumber: readinessInput.issueNumber, + summary: input.finalResponse, + checkResults: input.checkResults, + reviewSummary: input.review.summary, + }), + }, + ); + this.store.completeImplementationAttempt(input.attemptId, pullRequest.url); + return { + ...input, + pullRequestUrl: pullRequest.url, + queuedEventsSeen: this.store.countQueuedEvents(attempt.runId), + }; + } catch (error) { + this.store.failImplementationAttempt(input.attemptId, error); + throw error; + } + } +} diff --git a/src/workflows/implementation/steps/readiness/readiness.definition.ts b/src/workflows/implementation/steps/readiness/readiness.definition.ts new file mode 100644 index 0000000..55fb363 --- /dev/null +++ b/src/workflows/implementation/steps/readiness/readiness.definition.ts @@ -0,0 +1,85 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import type { ReadinessImplementation } from "./readiness.implementation"; + +export const readinessDecisionSchema = z + .object({ + ready: z.boolean(), + summary: z.string().min(1), + acceptanceCriteria: z.array(z.string().min(1)), + missingInformation: z.array(z.string().min(1)), + question: z.string().min(1).nullable(), + }) + .superRefine((decision, context) => { + if (decision.ready && decision.missingInformation.length > 0) { + context.addIssue({ + code: "custom", + message: "A ready issue cannot have missing information.", + }); + } + if (decision.ready && decision.question !== null) { + context.addIssue({ + code: "custom", + message: "A ready issue cannot ask a clarification question.", + }); + } + if (!decision.ready && decision.question === null) { + context.addIssue({ + code: "custom", + message: "An unready issue must ask a clarification question.", + }); + } + }); + +export type ReadinessDecision = z.infer; + +export const workflowInputSchema = z.object({ + controlRunId: z.string(), + correlationKey: z.string(), + queuedEventsSeen: z.number().int().nonnegative().default(0), +}); + +export const readinessOutputSchema = workflowInputSchema.extend({ + readiness: readinessDecisionSchema, + readinessEvaluationId: z.number().int().positive(), +}); + +const resumeSchema = z.object({ answer: z.string().min(1) }); +const suspendSchema = z.object({ + question: z.string(), + missingInformation: z.array(z.string()), + decision: readinessDecisionSchema, + evaluationId: z.number().int().positive(), +}); + +export function createReadinessStep( + implementation: Pick, +) { + return createStep({ + id: "readiness", + description: "Verify that the work item is sufficiently specified.", + inputSchema: workflowInputSchema, + outputSchema: readinessOutputSchema, + resumeSchema, + suspendSchema, + retries: 1, + execute: async ({ inputData, resumeData, suspend, abortSignal }) => { + const result = await implementation.execute({ + controlRunId: inputData.controlRunId, + ...(resumeData?.answer ? { resumeAnswer: resumeData.answer } : {}), + signal: abortSignal, + }); + if (!result.readiness.ready) { + return suspend({ + question: + result.readiness.question ?? + "What information is required before implementation can begin?", + missingInformation: result.readiness.missingInformation, + decision: result.readiness, + evaluationId: result.readinessEvaluationId, + }); + } + return { ...inputData, ...result }; + }, + }); +} diff --git a/src/workflows/implementation/steps/readiness/readiness.implementation.ts b/src/workflows/implementation/steps/readiness/readiness.implementation.ts new file mode 100644 index 0000000..3ae1476 --- /dev/null +++ b/src/workflows/implementation/steps/readiness/readiness.implementation.ts @@ -0,0 +1,46 @@ +import type { EventStore } from "../../../../persistence/event-store"; +import type { ReadinessEvaluator } from "../../../../services/readiness/contracts"; +import { PRODUCTION_GRAPH_VERSION } from "../../../definitions"; +import type { ReadinessDecision } from "./readiness.definition"; + +export class ReadinessImplementation { + constructor( + private readonly store: EventStore, + private readonly evaluator: ReadinessEvaluator, + private readonly graphVersion = PRODUCTION_GRAPH_VERSION, + ) {} + + async execute(input: { + controlRunId: string; + resumeAnswer?: string; + signal: AbortSignal; + }): Promise<{ readiness: ReadinessDecision; readinessEvaluationId: number }> { + const readinessInput = this.store.getReadinessInput(input.controlRunId); + if ( + input.resumeAnswer && + !readinessInput.clarifications.includes(input.resumeAnswer) + ) { + readinessInput.clarifications.push(input.resumeAnswer); + } + const evaluationId = this.store.startReadinessEvaluation( + input.controlRunId, + readinessInput, + this.evaluator.modelId, + this.evaluator.promptVersion, + this.graphVersion, + ); + try { + const evaluation = await this.evaluator.evaluate(readinessInput, { + abortSignal: input.signal, + }); + this.store.completeReadinessEvaluation(evaluationId, evaluation); + return { + readiness: evaluation.decision, + readinessEvaluationId: evaluationId, + }; + } catch (error) { + this.store.failReadinessEvaluation(evaluationId, error); + throw error; + } + } +} diff --git a/src/workflows/implementation/steps/reviewer/reviewer.definition.ts b/src/workflows/implementation/steps/reviewer/reviewer.definition.ts new file mode 100644 index 0000000..348831f --- /dev/null +++ b/src/workflows/implementation/steps/reviewer/reviewer.definition.ts @@ -0,0 +1,28 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { snapshotOutputSchema } from "../snapshot/snapshot.definition"; +import type { ReviewerImplementation } from "./reviewer.implementation"; + +export const reviewerOutputSchema = snapshotOutputSchema.extend({ + review: z.object({ + mode: z.literal("stub"), + decision: z.literal("approved"), + summary: z.string(), + }), +}); + +export type ReviewerOutput = z.infer; + +export function createReviewerStep( + implementation: Pick, +) { + return createStep({ + id: "stub-review", + inputSchema: snapshotOutputSchema, + outputSchema: reviewerOutputSchema, + execute: async ({ inputData }) => ({ + ...inputData, + ...(await implementation.execute(inputData)), + }), + }); +} diff --git a/src/workflows/implementation/steps/reviewer/reviewer.implementation.ts b/src/workflows/implementation/steps/reviewer/reviewer.implementation.ts new file mode 100644 index 0000000..d938523 --- /dev/null +++ b/src/workflows/implementation/steps/reviewer/reviewer.implementation.ts @@ -0,0 +1,19 @@ +import type { EventStore } from "../../../../persistence/event-store"; +import type { SnapshotOutput } from "../snapshot/snapshot.definition"; +import type { ReviewerOutput } from "./reviewer.definition"; + +export class ReviewerImplementation { + constructor(private readonly store: EventStore) {} + + async execute(input: SnapshotOutput): Promise { + this.store.setImplementationStage(input.attemptId, "reviewing"); + const review = { + mode: "stub" as const, + decision: "approved" as const, + summary: + "Temporary control-flow approval only. No independent specialist review was performed.", + }; + this.store.recordStubReview(input.attemptId, review); + return { ...input, review }; + } +} diff --git a/src/workflows/implementation/steps/snapshot/snapshot.definition.ts b/src/workflows/implementation/steps/snapshot/snapshot.definition.ts new file mode 100644 index 0000000..58f888e --- /dev/null +++ b/src/workflows/implementation/steps/snapshot/snapshot.definition.ts @@ -0,0 +1,24 @@ +import { createStep } from "@mastra/core/workflows"; +import { z } from "zod"; +import { deterministicChecksOutputSchema } from "../deterministic-checks/deterministic-checks.definition"; +import type { SnapshotImplementation } from "./snapshot.implementation"; + +export const snapshotOutputSchema = deterministicChecksOutputSchema.extend({ + commitSha: z.string(), +}); + +export type SnapshotOutput = z.infer; + +export function createSnapshotStep( + implementation: Pick, +) { + return createStep({ + id: "implementation-snapshot", + inputSchema: deterministicChecksOutputSchema, + outputSchema: snapshotOutputSchema, + execute: async ({ inputData, abortSignal }) => ({ + ...inputData, + ...(await implementation.execute(inputData, abortSignal)), + }), + }); +} diff --git a/src/workflows/implementation/steps/snapshot/snapshot.implementation.ts b/src/workflows/implementation/steps/snapshot/snapshot.implementation.ts new file mode 100644 index 0000000..c6cf909 --- /dev/null +++ b/src/workflows/implementation/steps/snapshot/snapshot.implementation.ts @@ -0,0 +1,38 @@ +import type { AppConfig } from "../../../../config"; +import type { EventStore } from "../../../../persistence/event-store"; +import { commitImplementationSnapshot } from "../../../../services/git/snapshot"; +import type { DeterministicChecksOutput } from "../deterministic-checks/deterministic-checks.definition"; +import type { SnapshotOutput } from "./snapshot.definition"; + +export class SnapshotImplementation { + constructor( + private readonly config: AppConfig["implementation"], + private readonly store: EventStore, + ) {} + + async execute( + input: DeterministicChecksOutput, + signal: AbortSignal, + ): Promise { + this.store.setImplementationStage(input.attemptId, "snapshotting"); + try { + const attempt = this.store.getImplementationAttempt(input.attemptId); + if (!attempt) { + throw new Error(`Implementation attempt ${input.attemptId} is missing.`); + } + const readinessInput = this.store.getReadinessInput(attempt.runId); + const commitSha = await commitImplementationSnapshot({ + worktreePath: input.worktreePath, + subject: `Implement #${readinessInput.issueNumber ?? "event"}: ${readinessInput.title}`, + authorName: this.config.gitAuthorName, + authorEmail: this.config.gitAuthorEmail, + signal, + }); + this.store.recordImplementationSnapshot(input.attemptId, commitSha); + return { ...input, commitSha }; + } catch (error) { + this.store.failImplementationAttempt(input.attemptId, error); + throw error; + } + } +} diff --git a/src/workflows/implementation/workflow.ts b/src/workflows/implementation/workflow.ts new file mode 100644 index 0000000..2c62787 --- /dev/null +++ b/src/workflows/implementation/workflow.ts @@ -0,0 +1,79 @@ +import { Mastra } from "@mastra/core/mastra"; +import { createWorkflow } from "@mastra/core/workflows"; +import { LibSQLStore } from "@mastra/libsql"; +import { PinoLogger } from "@mastra/loggers"; +import { MastraStorageExporter, Observability } from "@mastra/observability"; +import { createDemoImplementationWorkflow } from "../demo-implementation/workflow"; +import type { ImplementationWorkflowDependencies } from "./dependencies"; +import { createCodexImplementationStep } from "./steps/codex-implementation/codex-implementation.definition"; +import { createDeterministicChecksStep } from "./steps/deterministic-checks/deterministic-checks.definition"; +import { createPrepareWorktreeStep } from "./steps/prepare-worktree/prepare-worktree.definition"; +import { + createPublishPullRequestStep, + publishPullRequestOutputSchema, +} from "./steps/publish-pull-request/publish-pull-request.definition"; +import { + createReadinessStep, + workflowInputSchema, +} from "./steps/readiness/readiness.definition"; +import { createReviewerStep } from "./steps/reviewer/reviewer.definition"; +import { createSnapshotStep } from "./steps/snapshot/snapshot.definition"; + +export function createImplementationMastra( + dependencies: ImplementationWorkflowDependencies, +) { + const workflow = createWorkflow({ + id: "github-implementation", + inputSchema: workflowInputSchema, + outputSchema: publishPullRequestOutputSchema, + }) + .then(createReadinessStep(dependencies.implementations.readiness)) + .then( + createPrepareWorktreeStep(dependencies.implementations.prepareWorktree), + ) + .then( + createCodexImplementationStep( + dependencies.implementations.codexImplementation, + ), + ) + .then( + createDeterministicChecksStep( + dependencies.implementations.deterministicChecks, + ), + ) + .then(createSnapshotStep(dependencies.implementations.snapshot)) + .then(createReviewerStep(dependencies.implementations.reviewer)) + .then( + createPublishPullRequestStep( + dependencies.implementations.publishPullRequest, + ), + ) + .commit(); + const demoWorkflow = createDemoImplementationWorkflow( + dependencies.demoImplementations, + ); + + const storage = new LibSQLStore({ + id: "loop-storage", + url: dependencies.databaseUrl, + }); + const mastra = new Mastra({ + storage, + ...(dependencies.readinessAgent + ? { agents: { readiness: dependencies.readinessAgent } } + : {}), + workflows: { implementation: workflow, demoImplementation: demoWorkflow }, + logger: new PinoLogger({ name: "Implementer", level: "info" }), + observability: new Observability({ + configs: { + default: { + serviceName: "implementer", + exporters: [new MastraStorageExporter()], + logging: { enabled: true, level: "info" }, + }, + }, + }), + }); + + return { mastra, workflow, demoWorkflow, storage }; +} diff --git a/test/config.test.ts b/test/config.test.ts index bdbbe04..62a09f1 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -19,6 +19,48 @@ test("readiness model deployment settings come from the environment", () => { modelId: "test/model", timeoutMs: 45_000, }); + assert.deepEqual(config.implementation.model, { + baseUrl: "http://localhost:9999/v1", + apiKey: "test-key", + modelId: "test/model", + }); +}); + +test("the implementation model and worktree settings can be overridden", () => { + const config = loadConfig( + { + GITHUB_REPOSITORY: "example/app", + IMPLEMENTER_MODEL_BASE_URL: "http://localhost:8888/v1", + IMPLEMENTER_MODEL_API_KEY: "implementation-key", + IMPLEMENTER_MODEL: "implementation/model", + IMPLEMENTER_REPOSITORY_PATH: "source", + IMPLEMENTER_WORKTREE_ROOT: "worktrees", + IMPLEMENTER_BASE_BRANCH: "develop", + }, + "/tmp/project", + ); + + assert.equal(config.implementation.repository, "example/app"); + assert.equal(config.implementation.repositoryPath, "/tmp/project/source"); + assert.equal(config.implementation.worktreeRoot, "/tmp/project/worktrees"); + assert.equal(config.implementation.baseBranch, "develop"); + assert.deepEqual(config.implementation.model, { + baseUrl: "http://localhost:8888/v1", + apiKey: "implementation-key", + modelId: "implementation/model", + }); +}); + +test("the demo workflow must be selected explicitly", () => { + assert.equal(loadConfig({}, "/tmp/project").workflowMode, "production"); + assert.equal( + loadConfig({ IMPLEMENTER_WORKFLOW: "demo" }, "/tmp/project").workflowMode, + "demo", + ); + assert.throws( + () => loadConfig({ IMPLEMENTER_WORKFLOW: "unsafe" }, "/tmp/project"), + /must be either production or demo/, + ); }); test("GitHub App credentials are resolved from the environment", () => { @@ -37,6 +79,7 @@ test("GitHub App credentials are resolved from the environment", () => { installationId: "456", privateKeyPath: "/tmp/project/.secrets/app.pem", apiBaseUrl: "https://github.example/api/v3", + gitBaseUrl: "https://github.com", timeoutMs: 9_000, }); }); diff --git a/test/demo-implementation.test.ts b/test/demo-implementation.test.ts new file mode 100644 index 0000000..50416a3 --- /dev/null +++ b/test/demo-implementation.test.ts @@ -0,0 +1,161 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import test from "node:test"; +import { loadConfig } from "../src/config"; +import { EventStore } from "../src/persistence/event-store"; +import { normalizeGitHubEvent } from "../src/services/github/webhook"; +import type { + GitHubPullRequestPublisher, + PublishedPullRequest, +} from "../src/services/github/client"; +import { createDemoImplementationWorkflowImplementations } from "../src/workflows/demo-implementation/create-step-implementations"; + +class LocalPublisher implements GitHubPullRequestPublisher { + pushedBranch: string | null = null; + pullRequestBody: string | null = null; + + async fetchBase( + _repository: string, + repositoryPath: string, + baseBranch: string, + ): Promise { + execFileSync("git", ["update-ref", `refs/remotes/origin/${baseBranch}`, "HEAD"], { + cwd: repositoryPath, + }); + } + + async pushBranch( + _repository: string, + _worktreePath: string, + branch: string, + ): Promise { + this.pushedBranch = branch; + } + + async publishPullRequest( + _repository: string, + options: { head: string; base: string; title: string; body: string }, + ): Promise { + this.pullRequestBody = options.body; + return { number: 7, url: "https://github.test/example/app/pull/7" }; + } +} + +test("the demo path snapshots and publishes without running configured checks", async () => { + const root = path.join("/tmp", `demo-implementation-${randomUUID()}`); + const repositoryPath = path.join(root, "repository"); + fs.mkdirSync(repositoryPath, { recursive: true }); + execFileSync("git", ["init", "--initial-branch=main"], { cwd: repositoryPath }); + fs.writeFileSync( + path.join(repositoryPath, ".implementer.json"), + JSON.stringify({ + setup: [], + checks: [ + { + name: "must not run in demo", + command: [process.execPath, "-e", "process.exit(1)"], + timeoutMs: 10_000, + }, + ], + }), + ); + fs.writeFileSync(path.join(repositoryPath, "README.md"), "base\n"); + execFileSync("git", ["add", "."], { cwd: repositoryPath }); + execFileSync( + "git", + ["-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "base"], + { cwd: repositoryPath }, + ); + + const store = new EventStore(path.join(root, "events.sqlite")); + try { + const runId = store.ingest( + normalizeGitHubEvent({ + deliveryId: "demo-implementation", + eventName: "issues", + payload: { + action: "opened", + repository: { full_name: "example/app" }, + issue: { + number: 9, + title: "Add a demo result", + body: "Create DEMO.md", + labels: [], + }, + sender: { login: "alice" }, + }, + }), + "loop-bot", + ).runId!; + store.claimNextRun(); + const config = loadConfig( + { + IMPLEMENTER_WORKFLOW: "demo", + GITHUB_REPOSITORY: "example/app", + IMPLEMENTER_REPOSITORY_PATH: repositoryPath, + IMPLEMENTER_WORKTREE_ROOT: path.join(root, "worktrees"), + }, + root, + ); + const publisher = new LocalPublisher(); + const implementations = createDemoImplementationWorkflowImplementations({ + config: config.implementation, + store, + publisher, + readinessEvaluator: { + modelId: "unused", + promptVersion: "unused", + evaluate: async () => { + throw new Error("Readiness is outside this test."); + }, + }, + }); + const signal = new AbortController().signal; + const prepared = await implementations.prepareWorktree.execute({ + controlRunId: runId, + readiness: { + ready: true, + summary: "Ready", + acceptanceCriteria: ["DEMO.md exists"], + missingInformation: [], + question: null, + }, + signal, + }); + fs.writeFileSync(path.join(prepared.worktreePath, "DEMO.md"), "done\n"); + const published = await implementations.publishPullRequest.execute( + { + ...prepared, + controlRunId: runId, + correlationKey: "example/app#9", + queuedEventsSeen: 0, + readiness: { + ready: true, + summary: "Ready", + acceptanceCriteria: ["DEMO.md exists"], + missingInformation: [], + question: null, + }, + readinessEvaluationId: 1, + codexThreadId: "demo-thread", + finalResponse: "Created DEMO.md.", + }, + signal, + ); + + const attempt = store.getImplementationAttempt(prepared.attemptId); + assert.equal(attempt?.promptVersion, "codex-demo-no-goal-v1"); + assert.equal(attempt?.checkResults, null); + assert.equal(attempt?.status, "success"); + assert.equal(publisher.pushedBranch, prepared.branch); + assert.match(publisher.pullRequestBody ?? "", /reduced demo workflow/); + assert.match(publisher.pullRequestBody ?? "", /did not run/); + assert.equal(published.pullRequestUrl, "https://github.test/example/app/pull/7"); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/test/github-comments.test.ts b/test/github-comments.test.ts index 650db24..c703836 100644 --- a/test/github-comments.test.ts +++ b/test/github-comments.test.ts @@ -3,7 +3,19 @@ import { generateKeyPairSync, randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import test from "node:test"; -import { GitHubAppCommentPublisher } from "../src/github-comments"; +import { + GitHubAppCommentPublisher, + githubGitAuthorization, +} from "../src/services/github/client"; + +test("Git HTTPS uses GitHub App basic authentication", () => { + assert.equal( + githubGitAuthorization("installation-token"), + `Authorization: Basic ${Buffer.from( + "x-access-token:installation-token", + ).toString("base64")}`, + ); +}); test("the GitHub App client exchanges a signed JWT and posts an issue comment", async () => { const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); @@ -38,6 +50,7 @@ test("the GitHub App client exchanges a signed JWT and posts an issue comment", installationId: "456", privateKeyPath, apiBaseUrl: "https://api.github.test", + gitBaseUrl: "https://github.test", timeoutMs: 1_000, }, fetchMock, @@ -66,3 +79,56 @@ test("the GitHub App client exchanges a signed JWT and posts an issue comment", fs.rmSync(privateKeyPath, { force: true }); } }); + +test("pull-request publication reuses an existing open pull request", async () => { + const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const privateKeyPath = path.join("/tmp", `github-app-${randomUUID()}.pem`); + fs.writeFileSync( + privateKeyPath, + privateKey.export({ type: "pkcs8", format: "pem" }), + ); + const calls: Array<{ url: string; init: RequestInit }> = []; + const fetchMock = (async (input: string | URL | Request, init = {}) => { + const url = String(input); + calls.push({ url, init }); + if (url.endsWith("/app/installations/456/access_tokens")) { + return Response.json({ + token: "installation-token", + expires_at: new Date(Date.now() + 3_600_000).toISOString(), + }); + } + return Response.json([ + { number: 42, html_url: "https://github.test/example/app/pull/42" }, + ]); + }) as typeof fetch; + + try { + const publisher = new GitHubAppCommentPublisher( + { + appId: "123", + installationId: "456", + privateKeyPath, + apiBaseUrl: "https://api.github.test", + gitBaseUrl: "https://github.test", + timeoutMs: 1_000, + }, + fetchMock, + ); + const result = await publisher.publishPullRequest("example/app", { + head: "codex/issue-12-run", + base: "main", + title: "Implement issue", + body: "Evidence", + }); + + assert.deepEqual(result, { + number: 42, + url: "https://github.test/example/app/pull/42", + }); + assert.equal(calls.length, 2); + assert.match(calls[1]!.url, /pulls\?state=open/); + assert.equal(calls[1]!.init.method, undefined); + } finally { + fs.rmSync(privateKeyPath, { force: true }); + } +}); diff --git a/test/graph-definition.test.ts b/test/graph-definition.test.ts index 6a19559..9bc0323 100644 --- a/test/graph-definition.test.ts +++ b/test/graph-definition.test.ts @@ -1,12 +1,13 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + demoImplementationGraph, featureSuggestionGraph, productionGraph, refactoringGraph, reviewers, selfImprovementGraph, -} from "../src/graph-definition"; +} from "../src/workflows/definitions"; test("the graph contract retains every agreed independent reviewer", () => { assert.deepEqual( @@ -36,7 +37,11 @@ test("production and scheduled loops remain separate graphs with explicit status ); assert.equal( productionGraph.find(({ id }) => id === "codex-goal-implementation")?.status, - "planned", + "implemented", + ); + assert.equal( + productionGraph.find(({ id }) => id === "specialist-reviewers")?.status, + "partial", ); assert.equal( productionGraph.find(({ id }) => id === "readiness-and-decomposition") @@ -51,3 +56,20 @@ test("production and scheduled loops remain separate graphs with explicit status assert.ok(refactoringGraph.every(({ status }) => status === "planned")); assert.ok(featureSuggestionGraph.every(({ status }) => status === "planned")); }); + +test("the reduced demo graph is explicit about its missing assurance gates", () => { + assert.deepEqual( + demoImplementationGraph.map(({ id }) => id), + [ + "readiness", + "prepare-worktree", + "demo-codex-implementation", + "demo-pull-request", + ], + ); + assert.ok( + demoImplementationGraph + .find(({ id }) => id === "demo-pull-request") + ?.responsibility.includes("without deterministic checks or independent review"), + ); +}); diff --git a/test/implementation.test.ts b/test/implementation.test.ts new file mode 100644 index 0000000..f576e9b --- /dev/null +++ b/test/implementation.test.ts @@ -0,0 +1,184 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import test from "node:test"; +import { loadConfig } from "../src/config"; +import type { + GitHubPullRequestPublisher, + PublishedPullRequest, +} from "../src/services/github/client"; +import { createImplementationWorkflowImplementations } from "../src/workflows/implementation/create-step-implementations"; +import type { CodexImplementationOutput } from "../src/workflows/implementation/steps/codex-implementation/codex-implementation.definition"; +import { normalizeGitHubEvent } from "../src/services/github/webhook"; +import { EventStore } from "../src/persistence/event-store"; + +class LocalPublisher implements GitHubPullRequestPublisher { + pushedBranch: string | null = null; + pullRequestOptions: { + head: string; + base: string; + title: string; + body: string; + } | null = null; + + async fetchBase( + _repository: string, + repositoryPath: string, + baseBranch: string, + ): Promise { + execFileSync("git", ["update-ref", `refs/remotes/origin/${baseBranch}`, "HEAD"], { + cwd: repositoryPath, + }); + } + + async pushBranch( + _repository: string, + _worktreePath: string, + branch: string, + ): Promise { + this.pushedBranch = branch; + } + + async publishPullRequest( + _repository: string, + options: { head: string; base: string; title: string; body: string }, + ): Promise { + this.pullRequestOptions = options; + return { number: 42, url: "https://github.test/example/app/pull/42" }; + } +} + +test("Mastra owns the worktree, checks, snapshot, review stub, and PR handoff", async () => { + const root = path.join("/tmp", `implementation-service-${randomUUID()}`); + const repositoryPath = path.join(root, "repository"); + fs.mkdirSync(repositoryPath, { recursive: true }); + execFileSync("git", ["init", "--initial-branch=main"], { cwd: repositoryPath }); + fs.writeFileSync( + path.join(repositoryPath, ".implementer.json"), + JSON.stringify({ + setup: [], + checks: [ + { + name: "test check", + command: [process.execPath, "-e", "process.exit(0)"], + timeoutMs: 10_000, + }, + ], + }), + ); + fs.writeFileSync(path.join(repositoryPath, "README.md"), "base\n"); + execFileSync("git", ["add", "."], { cwd: repositoryPath }); + execFileSync( + "git", + [ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "-m", + "base", + ], + { cwd: repositoryPath }, + ); + + const store = new EventStore(path.join(root, "events.sqlite")); + try { + const runId = store.ingest( + normalizeGitHubEvent({ + deliveryId: "implementation-service", + eventName: "issues", + payload: { + action: "opened", + repository: { full_name: "example/app" }, + issue: { + number: 12, + title: "Add a result", + body: "Create RESULT.md", + labels: [], + }, + sender: { login: "alice" }, + }, + }), + "loop-bot", + ).runId!; + store.claimNextRun(); + const config = loadConfig( + { + GITHUB_REPOSITORY: "example/app", + IMPLEMENTER_REPOSITORY_PATH: repositoryPath, + IMPLEMENTER_WORKTREE_ROOT: path.join(root, "worktrees"), + }, + root, + ); + const publisher = new LocalPublisher(); + const implementations = createImplementationWorkflowImplementations({ + config: config.implementation, + store, + publisher, + readinessEvaluator: { + modelId: "unused", + promptVersion: "unused", + evaluate: async () => { + throw new Error("Readiness is outside this test."); + }, + }, + }); + const signal = new AbortController().signal; + const prepared = await implementations.prepareWorktree.execute({ + controlRunId: runId, + readiness: { + ready: true, + summary: "Ready", + acceptanceCriteria: ["RESULT.md exists"], + missingInformation: [], + question: null, + }, + signal, + }); + fs.writeFileSync(path.join(prepared.worktreePath, "RESULT.md"), "done\n"); + const implemented: CodexImplementationOutput = { + ...prepared, + controlRunId: runId, + correlationKey: "example/app#12", + queuedEventsSeen: 0, + readiness: { + ready: true, + summary: "Ready", + acceptanceCriteria: ["RESULT.md exists"], + missingInformation: [], + question: null, + }, + readinessEvaluationId: 1, + codexThreadId: "test-thread", + finalResponse: "Created RESULT.md.", + }; + const checked = await implementations.deterministicChecks.execute( + implemented, + signal, + ); + const snapshot = await implementations.snapshot.execute(checked, signal); + const reviewed = await implementations.reviewer.execute(snapshot); + const published = await implementations.publishPullRequest.execute( + reviewed, + signal, + ); + + assert.equal(checked.checkResults[0]?.exitCode, 0); + assert.equal(reviewed.review.mode, "stub"); + assert.match(reviewed.review.summary, /No independent specialist review/); + assert.equal(publisher.pushedBranch, prepared.branch); + assert.equal(publisher.pullRequestOptions?.head, prepared.branch); + assert.match(publisher.pullRequestOptions?.body ?? "", /test check/); + assert.equal( + published.pullRequestUrl, + "https://github.test/example/app/pull/42", + ); + assert.equal(store.getImplementationAttempt(prepared.attemptId)?.status, "success"); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/test/integration.test.ts b/test/integration.test.ts index cb4f45e..61b697c 100644 --- a/test/integration.test.ts +++ b/test/integration.test.ts @@ -7,13 +7,28 @@ import type { AppConfig } from "../src/config"; import type { GitHubCommentPublisher, PublishedGitHubComment, -} from "../src/github-comments"; -import type { ReadinessEvaluator } from "../src/readiness"; -import type { ReadinessDecision, ReadinessInput } from "../src/types"; +} from "../src/services/github/client"; +import type { + ReadinessEvaluator, + ReadinessInput, +} from "../src/services/readiness/contracts"; +import type { + CodexImplementationOutput, +} from "../src/workflows/implementation/steps/codex-implementation/codex-implementation.definition"; +import type { DeterministicChecksOutput } from "../src/workflows/implementation/steps/deterministic-checks/deterministic-checks.definition"; +import type { + PreparedWorktree, + PrepareWorktreeOutput, +} from "../src/workflows/implementation/steps/prepare-worktree/prepare-worktree.definition"; +import type { ReviewerOutput } from "../src/workflows/implementation/steps/reviewer/reviewer.definition"; +import type { SnapshotOutput } from "../src/workflows/implementation/steps/snapshot/snapshot.definition"; +import type { ReadinessDecision } from "../src/workflows/implementation/steps/readiness/readiness.definition"; +import type { DemoCodexImplementationOutput } from "../src/workflows/demo-implementation/steps/codex-implementation/codex-implementation.definition"; function config(implementationMs = 150): AppConfig { const id = randomUUID(); return { + workflowMode: "production", port: 0, databasePath: path.join("/tmp", `loop-events-${id}.sqlite`), mastraDatabaseUrl: `file:${path.join("/tmp", `loop-mastra-${id}.sqlite`)}`, @@ -23,7 +38,21 @@ function config(implementationMs = 150): AppConfig { githubOutboxRetryBaseMs: 10, githubOutboxMaxAttempts: 3, maxActiveImplementations: 1, - simulatedImplementationMs: implementationMs, + implementation: { + repository: "example/app", + repositoryPath: "/tmp/project", + baseBranch: "main", + worktreeRoot: "/tmp/worktrees", + checkConfigPath: ".implementer.json", + timeoutMs: 1_000, + model: { + baseUrl: "http://127.0.0.1:8888/v1", + apiKey: "local", + modelId: "test-implementer", + }, + gitAuthorName: "Test Bot", + gitAuthorEmail: "test@example.com", + }, readinessModel: { baseUrl: "http://127.0.0.1:8888/v1", apiKey: "local", @@ -50,6 +79,9 @@ function payload(issue: number, labels: string[] = []) { test("an event arriving during a run is attached without starting a second run", async () => { const app = await createApp(config(250), { readinessEvaluator: readyEvaluator(), + stepImplementations: fakeStepImplementations( + new FakeImplementationService(250), + ), }); try { const first = app.eventStore.ingest( @@ -75,8 +107,62 @@ test("an event arriving during a run is attached without starting a second run", } }); +test("demo mode routes webhook runs through the reduced workflow", async () => { + const appConfig = config(0); + appConfig.workflowMode = "demo"; + const service = new FakeImplementationService(); + let demoPublished = false; + const app = await createApp(appConfig, { + readinessEvaluator: readyEvaluator(), + stepImplementations: { + ...fakeStepImplementations(service), + codexImplementation: { + execute: async () => { + throw new Error("the production Codex step must not run in demo mode"); + }, + }, + }, + demoStepImplementations: { + prepareWorktree: { execute: () => service.prepare() }, + codexImplementation: { + execute: (input: PrepareWorktreeOutput) => service.implement(input), + }, + publishPullRequest: { + execute: async (input: DemoCodexImplementationOutput) => { + demoPublished = true; + return { + ...input, + commitSha: "demo-commit", + pullRequestUrl: "https://github.test/pull/demo", + }; + }, + }, + }, + }); + try { + const result = app.eventStore.ingest( + event("demo-route", "issues", payload(84)), + "loop-bot", + ); + app.coordinator.wake(); + await waitFor( + () => app.eventStore.getRun(result.runId!)?.status === "completed", + ); + assert.equal(demoPublished, true); + assert.equal( + app.eventStore.listReadinessEvaluations(result.runId!)[0]?.graphVersion, + "demo-v1", + ); + } finally { + await app.close(); + } +}); + test("a human comment resumes the suspended Mastra run", async () => { const app = await createApp(config(10), { + stepImplementations: fakeStepImplementations( + new FakeImplementationService(10), + ), readinessEvaluator: evaluator((input) => input.clarifications.length === 0 ? { @@ -139,6 +225,9 @@ test("a human comment resumes the suspended Mastra run", async () => { test("a suspended readiness run posts its question through the durable outbox", async () => { const publisher = new FakeGitHubCommentPublisher(); const app = await createApp(config(0), { + stepImplementations: fakeStepImplementations( + new FakeImplementationService(), + ), readinessEvaluator: evaluator(() => notReadyDecision()), githubCommentPublisher: publisher, }); @@ -168,6 +257,9 @@ test("a suspended readiness run posts its question through the durable outbox", test("GitHub comment delivery retries a transient API failure", async () => { const publisher = new FakeGitHubCommentPublisher(1); const app = await createApp(config(0), { + stepImplementations: fakeStepImplementations( + new FakeImplementationService(), + ), readinessEvaluator: evaluator(() => notReadyDecision()), githubCommentPublisher: publisher, }); @@ -194,6 +286,9 @@ test("GitHub comment delivery retries a transient API failure", async () => { test("restart recovery reconciles an existing GitHub comment instead of duplicating it", async () => { const appConfig = config(0); const firstApp = await createApp(appConfig, { + stepImplementations: fakeStepImplementations( + new FakeImplementationService(), + ), readinessEvaluator: evaluator(() => notReadyDecision()), }); const result = firstApp.eventStore.ingest( @@ -215,6 +310,9 @@ test("restart recovery reconciles an existing GitHub comment instead of duplicat body: `Recovered comment\n\n${pending.marker}`, }); const secondApp = await createApp(appConfig, { + stepImplementations: fakeStepImplementations( + new FakeImplementationService(), + ), readinessEvaluator: evaluator(() => notReadyDecision()), githubCommentPublisher: publisher, }); @@ -234,6 +332,9 @@ test("restart recovery reconciles an existing GitHub comment instead of duplicat test("the comment that starts a run cannot answer a question created later", async () => { const app = await createApp(config(0), { + stepImplementations: fakeStepImplementations( + new FakeImplementationService(), + ), readinessEvaluator: evaluator(() => notReadyDecision()), }); try { @@ -270,6 +371,9 @@ test("the comment that starts a run cannot answer a question created later", asy test("a transient readiness error is recorded and retried", async () => { let attempts = 0; const app = await createApp(config(0), { + stepImplementations: fakeStepImplementations( + new FakeImplementationService(), + ), readinessEvaluator: evaluator(() => { attempts += 1; if (attempts === 1) throw new Error("temporary model failure"); @@ -300,6 +404,9 @@ test("a transient readiness error is recorded and retried", async () => { test("an exhausted readiness retry fails with both attempts recorded", async () => { const app = await createApp(config(0), { + stepImplementations: fakeStepImplementations( + new FakeImplementationService(), + ), readinessEvaluator: evaluator(() => { throw new Error("model unavailable"); }), @@ -322,6 +429,38 @@ test("an exhausted readiness retry fails with both attempts recorded", async () } }); +test("a terminal implementation failure is posted through the durable outbox", async () => { + const publisher = new FakeGitHubCommentPublisher(); + const app = await createApp(config(0), { + readinessEvaluator: readyEvaluator(), + stepImplementations: fakeStepImplementations( + new FailingImplementationService(), + ), + githubCommentPublisher: publisher, + }); + try { + const result = app.eventStore.ingest( + event("implementation-failure", "issues", payload(31)), + "loop-bot", + ); + app.coordinator.wake(); + await waitFor( + () => + app.eventStore.listGitHubCommentOutbox(result.runId!)[0]?.status === + "sent", + ); + const run = app.eventStore.getRun(result.runId!); + const comment = app.eventStore.listGitHubCommentOutbox(result.runId!)[0]!; + assert.equal(run?.status, "failed"); + assert.equal(run?.lastError, "permission denied by automatic review"); + assert.match(comment.body, /Implementation stopped/); + assert.match(comment.body, /permission denied by automatic review/); + assert.match(comment.marker, /implementation:failed/); + } finally { + await app.close(); + } +}); + function event( deliveryId: string, eventName: string, @@ -366,7 +505,7 @@ function notReadyDecision(): ReadinessDecision { }; } -function readyEvaluator(): ReadinessEvaluator { +function readyEvaluator(): ReadinessEvaluator { return evaluator(() => readyDecision()); } @@ -374,7 +513,7 @@ function evaluator( evaluateDecision: ( input: ReadinessInput, ) => ReadinessDecision | Promise, -): ReadinessEvaluator { +): ReadinessEvaluator { return { modelId: "test-readiness", promptVersion: "readiness-test-v1", @@ -433,3 +572,89 @@ class FakeGitHubCommentPublisher implements GitHubCommentPublisher { return comment; } } + +class FakeImplementationService { + constructor(private readonly delayMs = 0) {} + + async prepare(): Promise { + return { + attemptId: 1, + worktreePath: "/tmp/worktree", + branch: "codex/test", + baseSha: "base", + goal: "Implement the test issue", + }; + } + + async implement( + input: PrepareWorktreeOutput, + ): Promise { + if (this.delayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, this.delayMs)); + } + return { + ...input, + codexThreadId: "thread-test", + finalResponse: "Implemented", + }; + } + + async check( + input: CodexImplementationOutput, + ): Promise { + return { ...input, checkResults: [] }; + } + + async snapshot( + input: DeterministicChecksOutput, + ): Promise { + return { ...input, commitSha: "commit" }; + } + + async review(input: SnapshotOutput): Promise { + return { + ...input, + review: { + mode: "stub", + decision: "approved", + summary: "Stub approval", + }, + }; + } + + async publish(input: ReviewerOutput) { + return { ...input, pullRequestUrl: "https://github.test/pull/1" }; + } +} + +class FailingImplementationService extends FakeImplementationService { + override async implement(): Promise { + throw new Error("permission denied by automatic review"); + } +} + +function fakeStepImplementations(service: FakeImplementationService) { + return { + prepareWorktree: { + execute: () => service.prepare(), + }, + codexImplementation: { + execute: (input: PrepareWorktreeOutput) => service.implement(input), + }, + deterministicChecks: { + execute: (input: CodexImplementationOutput) => service.check(input), + }, + snapshot: { + execute: (input: DeterministicChecksOutput) => service.snapshot(input), + }, + reviewer: { + execute: (input: SnapshotOutput) => service.review(input), + }, + publishPullRequest: { + execute: async (input: ReviewerOutput) => ({ + ...(await service.publish(input)), + queuedEventsSeen: 0, + }), + }, + }; +} diff --git a/test/readiness.test.ts b/test/readiness.test.ts index e0d4402..1d98293 100644 --- a/test/readiness.test.ts +++ b/test/readiness.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { readinessDecisionSchema } from "../src/readiness"; +import { readinessDecisionSchema } from "../src/workflows/implementation/steps/readiness/readiness.definition"; test("a ready decision cannot retain missing information", () => { const result = readinessDecisionSchema.safeParse({ diff --git a/test/store.test.ts b/test/store.test.ts index e45fe40..c8cd3f2 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -2,8 +2,11 @@ import assert from "node:assert/strict"; import { randomUUID } from "node:crypto"; import path from "node:path"; import test from "node:test"; -import { normalizeGitHubEvent, verifyGitHubSignature } from "../src/github"; -import { EventStore } from "../src/store"; +import { + normalizeGitHubEvent, + verifyGitHubSignature, +} from "../src/services/github/webhook"; +import { EventStore } from "../src/persistence/event-store"; import { createHmac } from "node:crypto"; function makeStore(): EventStore { @@ -101,3 +104,93 @@ test("signature verification uses the raw request body", () => { assert.equal(verifyGitHubSignature(body, signature, secret), true); assert.equal(verifyGitHubSignature(Buffer.from("changed"), signature, secret), false); }); + +test("implementation evidence remains reconstructable through publication", () => { + const store = makeStore(); + try { + const runId = store.ingest(event("implementation-evidence"), "loop-bot").runId!; + store.claimNextRun(); + const attemptId = store.startImplementationAttempt({ + runId, + repositoryPath: "/repo", + worktreePath: "/repo/.data/worktrees/run", + branch: "codex/issue-12-run", + goal: "Implement the accepted issue", + modelBaseUrl: "http://127.0.0.1:8888/v1", + modelId: "local/model", + promptVersion: "codex-goal-v1", + }); + const check = { + name: "tests", + command: ["pnpm", "test"], + timeoutMs: 10_000, + }; + store.configureImplementationAttempt(attemptId, "base-sha", [check]); + store.setCodexThreadId(attemptId, "thread-id"); + store.appendImplementationEvent(attemptId, { type: "thread.started" }); + store.completeCodexImplementation(attemptId, "Implemented the issue."); + store.recordImplementationCheck(attemptId, { + ...check, + exitCode: 0, + output: "ok", + durationMs: 25, + }); + store.recordImplementationSnapshot(attemptId, "commit-sha"); + store.recordStubReview(attemptId, { + mode: "stub", + decision: "approved", + }); + store.completeImplementationAttempt( + attemptId, + "https://github.test/example/app/pull/42", + ); + + const attempt = store.getImplementationAttempt(attemptId)!; + assert.equal(attempt.status, "success"); + assert.equal(attempt.stage, "completed"); + assert.equal(attempt.baseSha, "base-sha"); + assert.equal(attempt.codexThreadId, "thread-id"); + assert.equal(attempt.checkResults?.[0]?.exitCode, 0); + assert.equal(attempt.commitSha, "commit-sha"); + assert.equal(attempt.pullRequestUrl, "https://github.test/example/app/pull/42"); + } finally { + store.close(); + } +}); + +test("restart fails an interrupted implementation and queues an issue comment", () => { + const databasePath = path.join( + "/tmp", + `loop-store-restart-${randomUUID()}.sqlite`, + ); + const firstStore = new EventStore(databasePath); + const runId = firstStore.ingest(event("interrupted-run"), "loop-bot").runId!; + firstStore.claimNextRun(); + firstStore.startImplementationAttempt({ + runId, + repositoryPath: "/repo", + worktreePath: "/repo/.data/worktrees/run", + branch: "codex/issue-12-run", + goal: "Implement the accepted issue", + modelBaseUrl: "http://127.0.0.1:8888/v1", + modelId: "local/model", + promptVersion: "codex-goal-v1", + }); + firstStore.close(); + + const recoveredStore = new EventStore(databasePath); + try { + assert.equal(recoveredStore.getRun(runId)?.status, "failed"); + assert.equal( + recoveredStore.getLatestImplementationAttempt(runId)?.status, + "error", + ); + const comments = recoveredStore.listGitHubCommentOutbox(runId); + assert.equal(comments.length, 1); + assert.equal(comments[0]?.status, "pending"); + assert.match(comments[0]?.body ?? "", /stopped while this run was active/); + assert.match(comments[0]?.marker ?? "", /implementation:interrupted/); + } finally { + recoveredStore.close(); + } +});