From 3b5090a4909cab495d16fbb9d6c8562dc4205aa3 Mon Sep 17 00:00:00 2001 From: ZeroPoint95 <329227198+ZeroPoint95@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:58:50 -0400 Subject: [PATCH 1/3] senior-dev: a CMake default, and --verify-build/--verify-test for a check discovery cannot find MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A root CMakeLists.txt has always made a workspace accountable — both the build and the test demand apply — while no ecosystem default could name either, so a header-only CMake library failed verification with "no build entrypoint could be discovered" beside its own ctest suite. CMake is now the last ecosystem default: configure and build into .senior-dev/cmake-build, which no recorder reads, with one job per processor, then ctest there. For a project whose real check lives outside the folder (a fuzz harness, a benchmark's validation script), `codeaf senior-dev run` takes --verify-build and --verify-test. Each replaces discovery for its kind and is run by senior-dev itself on the frozen tree under the same strict preamble and ceiling. They come only from the command line, never from the working model, and naming one kind never excuses the other. Co-Authored-By: Claude Opus 5.5 --- internal/manual/chat/senior-dev.md | 36 ++++- internal/manual/chat_test.go | 27 ++++ internal/seniordev/app/args.go | 11 +- .../app/full_verification_declared_test.go | 120 ++++++++++++++++ .../seniordev/app/full_verification_run.go | 30 ++-- internal/seniordev/app/run.go | 7 + internal/seniordev/seniordev.go | 22 +-- .../session/fullverification/declared.go | 76 +++++++++++ .../session/fullverification/declared_test.go | 129 ++++++++++++++++++ .../session/fullverification/discovery.go | 27 ++++ 10 files changed, 465 insertions(+), 20 deletions(-) create mode 100644 internal/seniordev/app/full_verification_declared_test.go create mode 100644 internal/seniordev/session/fullverification/declared.go create mode 100644 internal/seniordev/session/fullverification/declared_test.go diff --git a/internal/manual/chat/senior-dev.md b/internal/manual/chat/senior-dev.md index faa085e6e5..615de1483a 100644 --- a/internal/manual/chat/senior-dev.md +++ b/internal/manual/chat/senior-dev.md @@ -466,6 +466,37 @@ resolve project imports. Older Python cannot use `-t .` on a folder without reports it ran no tests leaves the submission unchecked, even if it exits zero; the ending says `no tests were found by `. +## How does senior-dev find a project's build and tests — no build entrypoint could be discovered, CMake, ctest, --verify-build, --verify-test + +Any folder that looks like a project — a manifest, a `Makefile`, a +`CMakeLists.txt`, a `test/` folder — must have a build command and a test command +for its checks. senior-dev looks for them in CI, `AGENTS.md`, a Makefile or +`package.json` script, `README.md` or `CONTRIBUTING.md`, then a default for the +project's kind: Go, Rust, Maven, Gradle, .NET, Python, and CMake. + +For a CMake project with nothing else to go on, such as a header-only library, it +builds with `cmake -S . -B .senior-dev/cmake-build && cmake --build +.senior-dev/cmake-build --parallel` (one job per processor) and tests with +`ctest --test-dir .senior-dev/cmake-build --output-on-failure`. The build folder is +inside its own `.senior-dev` folder, so it never ends up in the change it hands in. +This needs `cmake` on the machine. + +When it finds nothing, the run fails its checks: the ending says `no build +entrypoint could be discovered` (or `test`), and the change is still handed in. + +**Naming the check yourself.** If the project is checked by something it cannot +find — a fuzzing harness or a benchmark's validation script outside the folder — +pass it on `run`: + +```sh +codeaf senior-dev run --verify-test '/scripts/validate.py --poc /tmp/poc' -- +``` + +`--verify-build CMD` and `--verify-test CMD` each replace what senior-dev would +have found. senior-dev runs them itself on the submitted tree, with the same strict +settings and time limit, and a non-zero exit fails the check. Naming one never +excuses the other. Only the command line sets them; senior-dev's own model cannot. + ## Can I run senior-dev in a folder that is not a git repo — a plain folder, no git, --in-place, operation not permitted, .Trash Yes. **senior-dev uses git only if it is there.** A folder with no git history — a plain @@ -1050,7 +1081,7 @@ after the run's record folder (such as `20260924-150405.000000`). That folder al `delegate-program.json`, with the instant senior-dev's process started and the instant it ended. -## senior-dev's flags — run, --variant, --in-place, --high, --max-cost +## senior-dev's flags — run, --variant, --in-place, --high, --verify-test, --max-cost `codeaf senior-dev ` is `codeaf senior-dev run -- `. codeaf gives every program it carries four flags: @@ -1073,6 +1104,9 @@ senior-dev's own flags on `run`: bare OpenRouter id, service-prefixed id, or short `/crew` model word; - `--asked` — the `--high` models were chosen by name, so one senior-dev cannot size ends the run before its first call rather than being skipped; +- `--verify-build CMD`, `--verify-test CMD` — the project's own build or test command, + run by senior-dev on the submitted tree instead of the one it would have found (see + how senior-dev finds a project's build and tests); - `--frontier` — accepted, and changes nothing: no call senior-dev makes uses that tier; - `--crew` — the models came from a conversation's crew: one its catalog cannot size is left out instead of failing the run. codeaf passes it with the crew's models. diff --git a/internal/manual/chat_test.go b/internal/manual/chat_test.go index 78707ddb60..eb6ace3fb5 100644 --- a/internal/manual/chat_test.go +++ b/internal/manual/chat_test.go @@ -3394,6 +3394,33 @@ func TestTheCallLogPageSaysWhyAFigureIsMissingFromARow(t *testing.T) { t.Fatalf("the question does not reach a section that says %q; a row missing a figure reads as a broken row", said) } +// A senior-dev run on a project whose build it could not find ends with a +// sentence a person pastes back as a question, and a person with a harness of +// their own asks how to hand it over. Both reach the section that names the +// flags, and the CMake shape reaches the section that names its default. +func TestTheUndiscoveredBuildQuestionsReachTheVerifyFlags(t *testing.T) { + for _, probe := range []struct { + asked string + says string + }{ + {"senior-dev says no build entrypoint could be discovered", "--verify-test"}, + {"how do I tell senior-dev which command runs my tests", "--verify-test"}, + {"senior-dev failed verification on my fuzz target harness", "--verify-build"}, + {"does senior-dev build a cmake project and run ctest", "cmake-build"}, + } { + found := false + for _, section := range Chat().Search(probe.asked, DefaultResults) { + if section.Page == "senior-dev" && strings.Contains(section.Body, probe.says) { + found = true + break + } + } + if !found { + t.Errorf("%q does not reach a senior-dev section that says %q", probe.asked, probe.says) + } + } +} + // A person whose reply vanished asks in their own words, and there is exactly // one page that can say who ended it. The pins are the two halves the fix owes // them: that a stop of their own is the only silent door, and that a thinking diff --git a/internal/seniordev/app/args.go b/internal/seniordev/app/args.go index 74b99bd807..a6658880f1 100644 --- a/internal/seniordev/app/args.go +++ b/internal/seniordev/app/args.go @@ -64,9 +64,14 @@ type cliArgs struct { // InPlace forces the snapshot recorder: senior-dev edits the workspace // without writing to any repository around it. Without it, the snapshot // recorder is still chosen wherever there is no git history to use. - InPlace bool - MaxCost *float64 - MaxHours *float64 + InPlace bool + // VerifyBuild and VerifyTest are the build and test the person named as + // the project's own check, each in place of discovery for its kind; empty + // leaves that kind to discovery (fullverification/declared.go). + VerifyBuild string + VerifyTest string + MaxCost *float64 + MaxHours *float64 } // CrewModel is a crew seat's model as a pool entry: the id filed under the diff --git a/internal/seniordev/app/full_verification_declared_test.go b/internal/seniordev/app/full_verification_declared_test.go new file mode 100644 index 0000000000..8d697e12e5 --- /dev/null +++ b/internal/seniordev/app/full_verification_declared_test.go @@ -0,0 +1,120 @@ +//go:build !windows + +package app + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/seniordev/session/fullverification" +) + +// These tests describe the project's own check named on the command line +// (`--verify-build`, `--verify-test`): the shape of a fuzz target or a +// benchmark rig, whose command lives beside the checkout rather than in it. + +func declaredVerificationRunner(t *testing.T, args cliArgs, files map[string]string) *pipeline { + t.Helper() + workspace := t.TempDir() + for name, content := range files { + if err := writeFile(filepath.Join(workspace, name), content); err != nil { + t.Fatal(err) + } + } + runner := newPipeline(args, workspace, pipelineDeps{ + Events: newEventWriter(io.Discard), Notes: io.Discard, + }) + t.Cleanup(runner.runtime.Close) + return runner +} + +// A DECLARED COMMAND IS RUN BY SENIOR-DEV ITSELF, IN PLACE OF DISCOVERY. The +// command here lives outside the workspace, as a harness's would, and leaves a +// mark to prove it ran; the Makefile's own red test is never run, because the +// person said what this project's test is. +func TestADeclaredTestRunsInPlaceOfTheDiscoveredOne(t *testing.T) { + harness := t.TempDir() + mark := filepath.Join(harness, "ran") + script := filepath.Join(harness, "validate.sh") + if err := os.WriteFile(script, []byte("#!/bin/sh\ntouch '"+mark+"'\n"), 0o755); err != nil { + t.Fatal(err) + } + runner := declaredVerificationRunner(t, cliArgs{VerifyTest: script}, map[string]string{ + "Makefile": "build:\n\t@true\ntest:\n\texit 1\n", + }) + verification := runner.runProjectVerification(context.Background()) + if verification.Failed != nil { + t.Fatalf("a green declared test failed verification: %#v (%s)", verification.Failed, verification.Failure) + } + if _, err := os.Stat(mark); err != nil { + t.Fatal("the declared command never ran") + } + if !soloVerificationPassed(&verification) { + t.Fatalf("a green build and a green declared test did not pass: %#v", verification) + } + sources := []string{} + for _, command := range verification.Commands { + evidence, _ := command.(map[string]any) + sources = append(sources, evidence["source"].(string)) + if evidence["cmd"] == "make test" { + t.Fatal("the discovered test ran although a test was declared") + } + } + if strings.Join(sources, ",") != "Makefile#build,"+fullverification.DeclaredTestSource { + t.Fatalf("sources = %v, want the discovered build and the declared test", sources) + } + if !strings.Contains(verification.Prompt, "named by the person who started this run") { + t.Fatalf("the evidence the model reads does not say where the check came from:\n%s", verification.Prompt) + } +} + +// A declared command is judged exactly as a discovered one: red is a failure, +// and it is the command's failure, never a missing-entrypoint one. +func TestARedDeclaredCommandFailsVerification(t *testing.T) { + runner := declaredVerificationRunner(t, cliArgs{VerifyBuild: "true", VerifyTest: "exit 3"}, + map[string]string{"NOTES.txt": "a harness-shaped project\n"}) + verification := runner.runProjectVerification(context.Background()) + if verification.Failed == nil { + t.Fatal("a declared test exiting 3 passed verification") + } + if verification.Failed.Source != fullverification.DeclaredTestSource || missingEntrypointFailure(verification) { + t.Fatalf("Failed = %#v, want the declared test's own failure", verification.Failed) + } +} + +// The strict preamble holds for a declared command too, so the person's +// command cannot be passed by a masked pipeline any more than a discovered one. +func TestADeclaredCommandRunsUnderTheStrictPreamble(t *testing.T) { + runner := declaredVerificationRunner(t, cliArgs{VerifyBuild: "true", VerifyTest: "false | cat"}, + map[string]string{"NOTES.txt": "a harness-shaped project\n"}) + if verification := runner.runProjectVerification(context.Background()); verification.Failed == nil { + t.Fatal("a pipeline whose first command fails passed verification") + } +} + +// THE 5C2 SHAPE. A header-only CMake library used to fail for want of a +// command; it now has CMake's own, and nothing in it is a missing entrypoint. +// The commands are not run here — this machine may have no cmake — only +// planned, which is where the old failure was decided. +func TestAHeaderOnlyCMakeLibraryIsNoLongerMissingAnEntrypoint(t *testing.T) { + runner := declaredVerificationRunner(t, cliArgs{}, map[string]string{ + "CMakeLists.txt": "cmake_minimum_required(VERSION 3.20)\nproject(lib CXX)\ninclude(CTest)\n", + "src/lib.hpp": "#pragma once\n", + "extras/tests/CMakeLists.txt": "add_test(NAME smoke COMMAND true)\n", + }) + plan := newProjectVerificationRun(runner, context.Background()).plan + if !planHasKind(plan, fullverification.KindBuild) || !planHasKind(plan, fullverification.KindTest) { + t.Fatalf("plan = %#v, want CMake's build and test", plan) + } + for _, entrypoint := range plan.Entrypoints { + // The build tree is the run's own folder, which no recorder reads, so + // configuring the project never puts a build directory in the answer. + if !strings.Contains(entrypoint.Command, seniorDevDataDirectory+"/") { + t.Fatalf("%q builds outside %s/", entrypoint.Command, seniorDevDataDirectory) + } + } +} diff --git a/internal/seniordev/app/full_verification_run.go b/internal/seniordev/app/full_verification_run.go index 33077ebc5d..6a9d212956 100644 --- a/internal/seniordev/app/full_verification_run.go +++ b/internal/seniordev/app/full_verification_run.go @@ -42,18 +42,32 @@ type verificationObservation struct { } func newProjectVerificationRun(runner *pipeline, ctx context.Context) *projectVerificationRun { + // What the person named with --verify-build and --verify-test replaces + // discovery for its kind (fullverification/declared.go). + plan := fullverification.Discover(runner.workspace).Declare(fullverification.Declared{ + Build: runner.args.VerifyBuild, Test: runner.args.VerifyTest, + }) + lines := []string{ + "# Independent full project verification", + "senior-dev independently discovered and ran the standard project entrypoints", + "below in fresh Bash subprocesses. These are process-derived command/exit", + "observations, not the model's claims. Consult them, but still run and cite", + "your own fresh verification commands.", + } + for _, entrypoint := range plan.Entrypoints { + if entrypoint.IsDeclared() { + lines = append(lines, "A command whose source is "+fullverification.DeclaredBuildSource+ + " or "+fullverification.DeclaredTestSource+" was named by the person who started this run as the", + "project's own check, in place of discovery; it is the one this run is judged by.") + break + } + } return &projectVerificationRun{ runner: runner, ctx: ctx, - plan: fullverification.Discover(runner.workspace), + plan: plan, result: projectVerificationResult{Commands: []any{}}, - lines: []string{ - "# Independent full project verification", - "senior-dev independently discovered and ran the standard project entrypoints", - "below in fresh Bash subprocesses. These are process-derived command/exit", - "observations, not the model's claims. Consult them, but still run and cite", - "your own fresh verification commands.", - }, + lines: lines, issues: []string{}, } } diff --git a/internal/seniordev/app/run.go b/internal/seniordev/app/run.go index 52c6998256..d8de06891b 100644 --- a/internal/seniordev/app/run.go +++ b/internal/seniordev/app/run.go @@ -72,6 +72,12 @@ type Options struct { // InPlace edits the folder without git: no commits, no refs, and the run's // checkpoints kept outside it. InPlace bool + // VerifyBuild and VerifyTest are the commands the person named as the + // project's own build and test (`--verify-build`, `--verify-test`). senior-dev + // runs each itself on the frozen tree in place of the one discovery would + // have chosen; empty leaves that kind to discovery. + VerifyBuild string + VerifyTest string // Crew says the pools came from the crew of the conversation that started // the run (`--crew`), not from a person typing them: a model the catalog // cannot size is dropped with a note, and a --high left empty routes on @@ -130,6 +136,7 @@ func runWith(ctx context.Context, host delegate.Host, options Options, notes io. args := cliArgs{ High: options.High, Low: options.Low, Frontier: options.Frontier, Variant: options.Variant, InPlace: options.InPlace, + VerifyBuild: options.VerifyBuild, VerifyTest: options.VerifyTest, } if len(splitPool(args.High)) == 0 { return refused("--high names no model, and the coder needs one to route on") diff --git a/internal/seniordev/seniordev.go b/internal/seniordev/seniordev.go index b8ebf81055..8f8bd2c0ff 100644 --- a/internal/seniordev/seniordev.go +++ b/internal/seniordev/seniordev.go @@ -186,16 +186,22 @@ func bindRun(fs *flag.FlagSet) delegate.Body { frontier := fs.String("frontier", "", "models for the frontier tier (no call uses it)") crew := fs.Bool("crew", false, "the models came from codeaf's crew: skip any it cannot size") asked := fs.Bool("asked", false, "the --high models were asked for by name; none is skipped") + // The project's own check, named by whoever started the run, for a project + // whose build or test discovery cannot find (fullverification/declared.go). + verifyBuild := fs.String("verify-build", "", "the project's build command, run on the frozen tree in place of the one discovered") + verifyTest := fs.String("verify-test", "", "the project's test command, run on the frozen tree in place of the one discovered") return func(ctx context.Context, host delegate.Host, args []string) error { run(ctx, host, app.Options{ - Goal: strings.Join(args, " "), - High: *high, - Low: *low, - Frontier: *frontier, - Variant: *variant, - InPlace: *inPlace, - Crew: *crew, - Asked: *asked, + Goal: strings.Join(args, " "), + High: *high, + Low: *low, + Frontier: *frontier, + Variant: *variant, + InPlace: *inPlace, + VerifyBuild: *verifyBuild, + VerifyTest: *verifyTest, + Crew: *crew, + Asked: *asked, }, os.Stderr) return nil } diff --git a/internal/seniordev/session/fullverification/declared.go b/internal/seniordev/session/fullverification/declared.go new file mode 100644 index 0000000000..635048ef84 --- /dev/null +++ b/internal/seniordev/session/fullverification/declared.go @@ -0,0 +1,76 @@ +//go:build !windows + +package fullverification + +import "strings" + +// THE PERSON WHO STARTED THE RUN MAY NAME THE PROJECT'S OWN CHECK. +// +// Discovery reads the workspace, and some projects keep the command that +// proves them somewhere it never looks: a fuzz target whose harness lives +// beside the checkout rather than in it, a benchmark rig with a validation +// script of its own, a build driven by a wrapper the README does not name. The +// floor demands a build and a test from every accountable project, so such a +// project failed verification for want of a command, with the command that +// would have proved it sitting one directory away. +// +// So `codeaf senior-dev run` takes `--verify-build` and `--verify-test`, and a +// command named there replaces discovery for its kind. It is still run by +// senior-dev itself, on the frozen tree, under the same strict preamble and +// ceiling as anything discovered; only WHERE the command came from changes. +// +// IT IS THE PERSON'S WORD, NEVER THE MODEL'S. The flags are read from the +// command line that started the run, which the model working the brief cannot +// write to; a check the working model could choose for itself is a check it +// could choose to pass. A declared kind is demanded like a discovered one, so +// naming a test never quietly excuses a missing build. + +// Declared is the build and test commands a person named on the command line. +// An empty field leaves that kind to discovery. +type Declared struct { + Build string + Test string +} + +// The sources a declared entrypoint carries, spelled as the flags a person +// typed, so every line that reports the command says where it came from. +const ( + DeclaredBuildSource = "--verify-build" + DeclaredTestSource = "--verify-test" +) + +// Declare returns the plan with each command the person named in place of +// whatever discovery chose for that kind. A declared kind is expected, so the +// floor holds the run to it exactly as it would a discovered one. +func (plan Plan) Declare(declared Declared) Plan { + build := strings.TrimSpace(declared.Build) + test := strings.TrimSpace(declared.Test) + if build == "" && test == "" { + return plan + } + chosen := map[EntrypointKind]Entrypoint{} + for _, entrypoint := range plan.Entrypoints { + chosen[entrypoint.Kind] = entrypoint + } + if build != "" { + chosen[KindBuild] = Entrypoint{Kind: KindBuild, Command: build, Source: DeclaredBuildSource} + plan.BuildExpected = true + } + if test != "" { + chosen[KindTest] = Entrypoint{Kind: KindTest, Command: test, Source: DeclaredTestSource} + plan.TestExpected = true + } + plan.Entrypoints = []Entrypoint{} + for _, kind := range []EntrypointKind{KindBuild, KindTest} { + if entrypoint, ok := chosen[kind]; ok { + plan.Entrypoints = append(plan.Entrypoints, entrypoint) + } + } + return plan +} + +// IsDeclared reports whether an entrypoint came from the command line rather +// than from discovery. +func (entrypoint Entrypoint) IsDeclared() bool { + return entrypoint.Source == DeclaredBuildSource || entrypoint.Source == DeclaredTestSource +} diff --git a/internal/seniordev/session/fullverification/declared_test.go b/internal/seniordev/session/fullverification/declared_test.go new file mode 100644 index 0000000000..41476ded01 --- /dev/null +++ b/internal/seniordev/session/fullverification/declared_test.go @@ -0,0 +1,129 @@ +//go:build !windows + +package fullverification + +import ( + "reflect" + "strings" + "testing" +) + +// A CMAKE PROJECT WITH NOTHING ELSE TO SAY IS GIVEN CMAKE'S OWN BUILD AND TEST. +// +// The shape is a header-only library: a root CMakeLists.txt, a test folder, +// and no CI, script or README command discovery recognizes. CMakeLists.txt +// makes it accountable, so both demands apply; before the CMake default there +// was nothing to meet either with, and every run on it failed verification +// with "no build entrypoint could be discovered". +func TestACMakeProjectIsGivenCMakesOwnBuildAndTest(t *testing.T) { + workspace := t.TempDir() + writeDiscoveryFile(t, workspace, "CMakeLists.txt", "cmake_minimum_required(VERSION 3.20)\nproject(demo CXX)\ninclude(CTest)\n") + writeDiscoveryFile(t, workspace, "extras/tests/CMakeLists.txt", "add_test(NAME demo COMMAND true)\n") + plan := Discover(workspace) + if !plan.BuildExpected || !plan.TestExpected { + t.Fatalf("plan = %#v, want a CMake project held to both demands", plan) + } + want := []Entrypoint{ + {Kind: KindBuild, Command: "cmake -S . -B .senior-dev/cmake-build && cmake --build .senior-dev/cmake-build --parallel \"$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)\"", Source: "CMakeLists.txt"}, + {Kind: KindTest, Command: "ctest --test-dir .senior-dev/cmake-build --output-on-failure", Source: "CMakeLists.txt"}, + } + if !reflect.DeepEqual(plan.Entrypoints, want) { + t.Fatalf("plan = %#v, want %#v", plan.Entrypoints, want) + } +} + +// The CMake default is the LAST ecosystem asked, so a project that already +// had a default keeps exactly the one it had, and a CMake project whose own +// files name a command keeps that command. +func TestTheCMakeDefaultNeverDisplacesAnEarlierChoice(t *testing.T) { + goWorkspace := t.TempDir() + writeDiscoveryFile(t, goWorkspace, "go.mod", "module example.test/demo\n") + writeDiscoveryFile(t, goWorkspace, "CMakeLists.txt", "project(demo C)\n") + plan := Discover(goWorkspace) + for _, entrypoint := range plan.Entrypoints { + if entrypoint.Source == "CMakeLists.txt" { + t.Fatalf("a Go module with a CMakeLists.txt lost its Go default: %#v", plan.Entrypoints) + } + } + + pythonWorkspace := t.TempDir() + writeDiscoveryFile(t, pythonWorkspace, "pyproject.toml", "[project]\nname = \"demo\"\n") + writeDiscoveryFile(t, pythonWorkspace, "pytest.ini", "[pytest]\n") + writeDiscoveryFile(t, pythonWorkspace, "tests/test_one.py", "") + writeDiscoveryFile(t, pythonWorkspace, "CMakeLists.txt", "project(ext C)\n") + plan = Discover(pythonWorkspace) + for _, entrypoint := range plan.Entrypoints { + if entrypoint.Source == "CMakeLists.txt" { + t.Fatalf("a Python project with a native extension lost its pytest default: %#v", plan.Entrypoints) + } + } + + documented := t.TempDir() + writeDiscoveryFile(t, documented, "CMakeLists.txt", "project(demo CXX)\n") + writeDiscoveryFile(t, documented, "Makefile", "build:\n\tcmake --build out\ntest:\n\tcd out && ctest\n") + plan = Discover(documented) + want := []Entrypoint{ + {Kind: KindBuild, Command: "make build", Source: "Makefile#build"}, + {Kind: KindTest, Command: "make test", Source: "Makefile#test"}, + } + if !reflect.DeepEqual(plan.Entrypoints, want) { + t.Fatalf("plan = %#v, want the Makefile's own targets kept over the CMake default", plan.Entrypoints) + } +} + +func TestADeclaredCommandReplacesDiscoveryForItsKindOnly(t *testing.T) { + workspace := t.TempDir() + writeDiscoveryFile(t, workspace, "go.mod", "module example.test/demo\n") + plan := Discover(workspace).Declare(Declared{Test: " /scripts/validate.py --poc /tmp/poc "}) + want := []Entrypoint{ + {Kind: KindBuild, Command: "go build ./...", Source: "go.mod"}, + {Kind: KindTest, Command: "/scripts/validate.py --poc /tmp/poc", Source: DeclaredTestSource}, + } + if !reflect.DeepEqual(plan.Entrypoints, want) { + t.Fatalf("plan = %#v, want %#v", plan.Entrypoints, want) + } + if !plan.Entrypoints[1].IsDeclared() || plan.Entrypoints[0].IsDeclared() { + t.Fatalf("IsDeclared must name exactly the declared entrypoint: %#v", plan.Entrypoints) + } +} + +// A DECLARED KIND IS A DEMAND, AND DECLARING ONE NEVER EXCUSES THE OTHER. A +// workspace discovery called unaccountable is held to what the person named, +// and nothing more; an accountable one still owes the kind they did not name. +func TestADeclaredKindIsDemandedAndTheOtherIsLeftToDiscovery(t *testing.T) { + bare := t.TempDir() + writeDiscoveryFile(t, bare, "NOTES.txt", "nothing to build\n") + plan := Discover(bare).Declare(Declared{Test: "./check.sh"}) + if !plan.TestExpected || plan.BuildExpected { + t.Fatalf("plan = %#v, want the declared test demanded and no build invented", plan) + } + + accountable := t.TempDir() + writeDiscoveryFile(t, accountable, "Makefile", "lint:\n\techo lint\n") + plan = Discover(accountable).Declare(Declared{Test: "./check.sh"}) + if !plan.BuildExpected { + t.Fatalf("plan = %#v, want the undeclared build still demanded", plan) + } + for _, entrypoint := range plan.Entrypoints { + if entrypoint.Kind == KindBuild { + t.Fatalf("plan = %#v, want no build entrypoint in this fixture", plan) + } + } +} + +func TestDeclaringNothingLeavesThePlanAsDiscovered(t *testing.T) { + workspace := t.TempDir() + writeDiscoveryFile(t, workspace, "go.mod", "module example.test/demo\n") + discovered := Discover(workspace) + if declared := discovered.Declare(Declared{Build: " ", Test: "\n"}); !reflect.DeepEqual(declared, discovered) { + t.Fatalf("blank declarations changed the plan: %#v, want %#v", declared, discovered) + } +} + +// The CMake build tree must be senior-dev's own folder, never a directory the +// project could ship. The app package pins the same against its own constant. +func TestTheCMakeBuildTreeIsInsideSeniorDevsOwnFolder(t *testing.T) { + if !strings.HasPrefix(cmakeBuildDirectory, ".senior-dev/") { + t.Fatalf("cmakeBuildDirectory = %q, want it under .senior-dev/", cmakeBuildDirectory) + } +} diff --git a/internal/seniordev/session/fullverification/discovery.go b/internal/seniordev/session/fullverification/discovery.go index 5868ebb483..b2c885be4f 100644 --- a/internal/seniordev/session/fullverification/discovery.go +++ b/internal/seniordev/session/fullverification/discovery.go @@ -552,10 +552,37 @@ func ecosystemDefaults(workspace string) []Entrypoint { add(KindTest, unittestDiscoveryCommand(workspace), "Python test files") } } + case fileExists(filepath.Join(workspace, "CMakeLists.txt")): + // A CMAKE PROJECT IS HELD TO THE FLOOR, SO IT IS GIVEN A WAY TO MEET IT. + // CMakeLists.txt has always made a workspace accountable, which demands + // a build and a test, while nothing here could name either: a project + // whose CI and README say no recognized command — a header-only library, + // most of them — failed verification for want of a command nobody + // looked for, with its own ctest suite sitting in the tree. It comes + // after every other ecosystem, so a project that already had a default + // keeps exactly the one it had. + // + // The build tree is senior-dev's own folder, which neither recorder + // nor the fingerprint ever reads, so configuring a project never puts + // a build directory into the answer and a later pass builds on the last + // one instead of from nothing. + // + // THE BUILD IS AS WIDE AS THE MACHINE, AND NO WIDER. A bare --parallel + // hands Makefiles an unlimited -j, which on ArduinoJson's test suite + // started every compiler at once and had them killed for memory; a + // serial build of the same suite does not fit the verification ceiling. + add(KindBuild, "cmake -S . -B "+cmakeBuildDirectory+" && cmake --build "+cmakeBuildDirectory+ + ` --parallel "$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)"`, "CMakeLists.txt") + add(KindTest, "ctest --test-dir "+cmakeBuildDirectory+" --output-on-failure", "CMakeLists.txt") } return entries } +// cmakeBuildDirectory is where the CMake default configures and builds, +// relative to the workspace: inside senior-dev's own folder, which is never +// part of the tree a run compares, freezes, restores or ships. +const cmakeBuildDirectory = ".senior-dev/cmake-build" + // unittestDiscoveryCommand names each top-level test folder that actually // holds tests. Python 3.10 cannot use -t . for a folder without __init__.py: // omitting -t in that case still puts the project root on sys.path and runs From 0e1c6087a7c9da94e1db2c265d03903469fb902e Mon Sep 17 00:00:00 2001 From: ZeroPoint95 <329227198+ZeroPoint95@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:59:20 -0400 Subject: [PATCH 2/3] changes: the change entry for #1793 Co-Authored-By: Claude Opus 5.5 --- .../1793-senior-dev-cmake-verify-command.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/changes/unreleased/1793-senior-dev-cmake-verify-command.md diff --git a/docs/changes/unreleased/1793-senior-dev-cmake-verify-command.md b/docs/changes/unreleased/1793-senior-dev-cmake-verify-command.md new file mode 100644 index 0000000000..fa322f1b67 --- /dev/null +++ b/docs/changes/unreleased/1793-senior-dev-cmake-verify-command.md @@ -0,0 +1,15 @@ +--- +kind: added +title: senior-dev checks CMake projects itself and takes --verify-build and --verify-test +pr: 1793 +surface: [engine, docs] +invalidates: + - "A CMake project with no recognized CI, script or README command failed senior-dev's checks with `no build entrypoint could be discovered`. senior-dev now builds it into `.senior-dev/cmake-build` and runs `ctest` there." + - "There was no way to tell senior-dev which command checks a project; `submission_evidence` was the only channel, and nothing read it. `codeaf senior-dev run` takes `--verify-build CMD` and `--verify-test CMD`, which senior-dev runs itself on the submitted tree in place of discovery." +--- + +The shape is a header-only library or a fuzz target whose harness lives beside +the checkout: CyberGym's arvo_24633 was fixed, verified externally and graded +4/4, yet senior-dev called it a failed change. The flags come only from the +command line, never from the working model, so a check it could choose is not a +check it could choose to pass; naming one kind never excuses the other. From a5ae7fa7959a73cec8aaba4d92c1ab3b528fbadb Mon Sep 17 00:00:00 2001 From: ZeroPoint95 <329227198+ZeroPoint95@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:00:25 -0400 Subject: [PATCH 3/3] senior-dev: the CMake test default configures and builds its own tree Discover picks each kind on its own, so a CMake project whose README or Makefile names a build and no test was paired with that build and `ctest --test-dir .senior-dev/cmake-build`, a folder nothing configured: the run reported the project's tests red where it used to say plainly that no test entrypoint could be discovered. The test default now configures and builds the tree first, as every other ecosystem's test default stands alone, and runs ctest from inside it, which works before CMake 3.20. Co-Authored-By: Claude Opus 5.5 --- internal/manual/chat/senior-dev.md | 8 +++-- .../session/fullverification/declared_test.go | 36 +++++++++++++++++-- .../session/fullverification/discovery.go | 20 +++++++++-- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/internal/manual/chat/senior-dev.md b/internal/manual/chat/senior-dev.md index 615de1483a..f5ac93ee96 100644 --- a/internal/manual/chat/senior-dev.md +++ b/internal/manual/chat/senior-dev.md @@ -476,9 +476,11 @@ project's kind: Go, Rust, Maven, Gradle, .NET, Python, and CMake. For a CMake project with nothing else to go on, such as a header-only library, it builds with `cmake -S . -B .senior-dev/cmake-build && cmake --build -.senior-dev/cmake-build --parallel` (one job per processor) and tests with -`ctest --test-dir .senior-dev/cmake-build --output-on-failure`. The build folder is -inside its own `.senior-dev` folder, so it never ends up in the change it hands in. +.senior-dev/cmake-build --parallel` (one job per processor). It tests by running +the same configure and build first, then `ctest --output-on-failure` in that +folder, so the test still works when the build command came from somewhere else. +The build folder is inside its own `.senior-dev` folder, so it never ends up in +the change it hands in. This needs `cmake` on the machine. When it finds nothing, the run fails its checks: the ending says `no build diff --git a/internal/seniordev/session/fullverification/declared_test.go b/internal/seniordev/session/fullverification/declared_test.go index 41476ded01..619c6e56ca 100644 --- a/internal/seniordev/session/fullverification/declared_test.go +++ b/internal/seniordev/session/fullverification/declared_test.go @@ -23,15 +23,47 @@ func TestACMakeProjectIsGivenCMakesOwnBuildAndTest(t *testing.T) { if !plan.BuildExpected || !plan.TestExpected { t.Fatalf("plan = %#v, want a CMake project held to both demands", plan) } + const build = "cmake -S . -B .senior-dev/cmake-build && cmake --build .senior-dev/cmake-build --parallel \"$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)\"" want := []Entrypoint{ - {Kind: KindBuild, Command: "cmake -S . -B .senior-dev/cmake-build && cmake --build .senior-dev/cmake-build --parallel \"$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)\"", Source: "CMakeLists.txt"}, - {Kind: KindTest, Command: "ctest --test-dir .senior-dev/cmake-build --output-on-failure", Source: "CMakeLists.txt"}, + {Kind: KindBuild, Command: build, Source: "CMakeLists.txt"}, + {Kind: KindTest, Command: build + " && cd .senior-dev/cmake-build && ctest --output-on-failure", Source: "CMakeLists.txt"}, } if !reflect.DeepEqual(plan.Entrypoints, want) { t.Fatalf("plan = %#v, want %#v", plan.Entrypoints, want) } } +// THE CMAKE TEST STANDS ALONE. Discover picks each kind on its own, so a CMake +// project whose README or Makefile names a build and no test is paired with +// that build and the CMake test. A test that assumed the CMake build had run +// would fail on a folder nothing configured and report the project's tests +// red; the test default configures and builds its own tree first. +func TestTheCMakeTestBuildsItsOwnTreeWhenTheBuildCameFromElsewhere(t *testing.T) { + for _, fixture := range []struct { + name, file, body string + build Entrypoint + }{ + {"README build", "README.md", "Build with `cmake --build build`.\n", + Entrypoint{Kind: KindBuild, Command: "cmake --build build", Source: "README.md"}}, + {"Makefile build target", "Makefile", "build:\n\tcmake --build out\n", + Entrypoint{Kind: KindBuild, Command: "make build", Source: "Makefile#build"}}, + } { + t.Run(fixture.name, func(t *testing.T) { + workspace := t.TempDir() + writeDiscoveryFile(t, workspace, "CMakeLists.txt", "project(demo CXX)\ninclude(CTest)\n") + writeDiscoveryFile(t, workspace, fixture.file, fixture.body) + want := []Entrypoint{fixture.build, { + Kind: KindTest, + Command: cmakeConfigureAndBuild + " && cd " + cmakeBuildDirectory + " && ctest --output-on-failure", + Source: "CMakeLists.txt", + }} + if plan := Discover(workspace); !reflect.DeepEqual(plan.Entrypoints, want) { + t.Fatalf("plan = %#v, want %#v", plan.Entrypoints, want) + } + }) + } +} + // The CMake default is the LAST ecosystem asked, so a project that already // had a default keeps exactly the one it had, and a CMake project whose own // files name a command keeps that command. diff --git a/internal/seniordev/session/fullverification/discovery.go b/internal/seniordev/session/fullverification/discovery.go index b2c885be4f..1b895f550c 100644 --- a/internal/seniordev/session/fullverification/discovery.go +++ b/internal/seniordev/session/fullverification/discovery.go @@ -571,13 +571,27 @@ func ecosystemDefaults(workspace string) []Entrypoint { // hands Makefiles an unlimited -j, which on ArduinoJson's test suite // started every compiler at once and had them killed for memory; a // serial build of the same suite does not fit the verification ceiling. - add(KindBuild, "cmake -S . -B "+cmakeBuildDirectory+" && cmake --build "+cmakeBuildDirectory+ - ` --parallel "$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)"`, "CMakeLists.txt") - add(KindTest, "ctest --test-dir "+cmakeBuildDirectory+" --output-on-failure", "CMakeLists.txt") + // + // THE TEST BUILDS ITS OWN TREE, as every other ecosystem's test default + // does (`go test` compiles, `mvn test` packages). Discover picks each kind + // on its own, so a CMake project whose README names a build and no test + // is paired with that build and this test; a ctest that assumed the + // default build had run would fail on a directory nothing configured, + // and the run would chase a red test that is really a missing folder. + // When the default build did run, configuring and building again is an + // incremental no-op. ctest runs from inside the tree rather than with + // --test-dir, which needs CMake 3.20 and older distributions do not ship. + add(KindBuild, cmakeConfigureAndBuild, "CMakeLists.txt") + add(KindTest, cmakeConfigureAndBuild+" && cd "+cmakeBuildDirectory+" && ctest --output-on-failure", "CMakeLists.txt") } return entries } +// cmakeConfigureAndBuild configures and builds the CMake project in +// [cmakeBuildDirectory], one job per processor. +const cmakeConfigureAndBuild = "cmake -S . -B " + cmakeBuildDirectory + " && cmake --build " + cmakeBuildDirectory + + ` --parallel "$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)"` + // cmakeBuildDirectory is where the CMake default configures and builds, // relative to the workspace: inside senior-dev's own folder, which is never // part of the tree a run compares, freezes, restores or ships.