diff --git a/.github/workflows/pool-mirror.yml b/.github/workflows/pool-mirror.yml new file mode 100644 index 0000000000..8dd9b92489 --- /dev/null +++ b/.github/workflows/pool-mirror.yml @@ -0,0 +1,95 @@ +name: Model Pool mirror + +# Mirrors the relay's signed index document onto the `model-pool` orphan data +# branch, under pool/index.json and pool/index.json.sig, so it can be served +# and linked from GitHub. The branch holds the signed document and its +# signature, and nothing derived from them — no count, no leaderboard, no +# extracted statistic. +on: + schedule: + # Every six hours. + - cron: '0 */6 * * *' + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: pool-mirror + cancel-in-progress: false + +jobs: + mirror: + runs-on: ubuntu-latest + steps: + # Gets relay/verify.mjs, the only thing this workflow needs from the tree. + - uses: actions/checkout@v4 + + # -fsS: fail the job on any non-200, print an error on any other failure. + - name: Fetch the signed document and its signature + run: | + set -euo pipefail + curl -fsS -o index.json https://codeaf.agentfield.ai/pool/index.json + curl -fsS -o index.json.sig https://codeaf.agentfield.ai/pool/index.json.sig + + # The value is the public key copied from relay/wrangler.toml, and it is a + # literal: no secret is needed to verify a signature. A bad signature or + # any non-2xx fetch fails the job before anything is published. + - name: Verify the signature + env: + POOL_PUBLIC_KEY: WOAo+g/oKxAV9vVqv2Q14w1TyyyiouwFO2fC0zgcps0= + run: node relay/verify.mjs index.json index.json.sig "$POOL_PUBLIC_KEY" + + - name: Mirror onto the model-pool branch + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p mirror + cd mirror + git init -q + git remote add origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # The data branch shares no history with the tree; adopt it when it + # exists and start an orphan when this is the first run. + if git fetch -q origin model-pool; then + git checkout -q -B model-pool FETCH_HEAD + else + git checkout -q --orphan model-pool + fi + + mkdir -p pool + if [ ! -f pool/README.md ]; then + cat > pool/README.md <<'EOF' + # model-pool + + This orphan branch holds the Model Pool's published index document and + its detached signature, and nothing derived from them. + + - `pool/index.json` — the signed index document. + - `pool/index.json.sig` — the detached Ed25519 signature (standard + base64) over the exact bytes of `pool/index.json`. + + Both files are copies of what the relay serves; there is no count, no + leaderboard and no extracted statistic here. Verify them with: + + ``` + node relay/verify.mjs pool/index.json pool/index.json.sig + ``` + EOF + fi + + cp ../index.json pool/index.json + cp ../index.json.sig pool/index.json.sig + + git add -A + if git diff --quiet && git diff --cached --quiet; then + echo "No change; nothing to publish." + exit 0 + fi + + version=$(node -e "process.stdout.write(String(JSON.parse(require('fs').readFileSync('../index.json', 'utf8')).version))") + git commit -q -m "pool: index version ${version}" + git push -q origin model-pool diff --git a/README.md b/README.md index 6bde91a003..07d8b2fd2b 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,22 @@ request goes to the provider that has been fastest for that kind of call. Providers built in: OpenRouter, DeepSeek, GLM, Kimi, MiniMax, Qwen, Ollama and any OpenAI-compatible endpoint. +## Model Pool + +The picker can choose models from what other installs have found. It is on by +default: what an install sends is computed, text-free numbers about the models +it ran (role, model, a number, which model judged, door, size bucket, day) under +a per-install nonce, never code, +prompts, paths or an identity, and `codeaf pool status` shows exactly what is +waiting to go. Turn it off with `model_pool = off` on the settings sheet or +`CODEAF_MODEL_POOL=off`; `read` uses the pool and sends nothing. The relay +publishes a signed index the crew picker reads under `picked from = learn`. The index is mirrored on the `model-pool` branch at +`pool/index.json`. The design is [Pareto Crewing](docs/design/model-pool/pareto-crewing.pdf); +the relay's code is under `relay/`, with a [runbook](docs/design/model-pool/RUNBOOK.md) +that includes running your own. + +[![pool updated](https://img.shields.io/github/last-commit/Agent-Field/CodeAF/model-pool?label=pool%20updated)](https://github.com/Agent-Field/CodeAF/tree/model-pool) + ## Standing orders Rules, reminders and watches are one thing, and you set them up by saying them. diff --git a/cmd/codeaf/chat.go b/cmd/codeaf/chat.go index 151ec64c87..e5c31a9b55 100644 --- a/cmd/codeaf/chat.go +++ b/cmd/codeaf/chat.go @@ -166,6 +166,15 @@ func buildBrain(w *chatWindow, session string, opts brainOptions) (*chatBrain, e // then let the machine form print underneath, which said one fact twice. return nil, err } + // A tier row that says auto is answered from this catalog (config.AutoModels), + // so the word is wired BEFORE the seats handed in are applied — a door whose + // ladder answered the word before this line resolved it from nothing. The + // read is the same non-blocking one, never a fetch. + modelCatalog := catalog.LoadLazy(context.Background(), catalog.Options{ + BaseURL: settings.BaseURL, APIKey: settings.APIKey, Dir: settings.ProfileDir, + }) + config.AutoModels = modelCatalog.ModelsNow + wirePoolIndex(settings.ProfileDir) if opts.seats != nil { applySeats(&settings, *opts.seats) } else { @@ -180,9 +189,6 @@ func buildBrain(w *chatWindow, session string, opts brainOptions) (*chatBrain, e // to a tool the user has not been able to reach yet — the surface has not // been drawn. Asking it in front of the first frame buys nothing and can // cost fifteen seconds of dead terminal. - modelCatalog := catalog.LoadLazy(context.Background(), catalog.Options{ - BaseURL: settings.BaseURL, APIKey: settings.APIKey, Dir: settings.ProfileDir, - }) // Every client built below shapes its requests against these rows: which // knobs a model accepts is the catalog's answer, not a guess. settings.Models = modelCatalog @@ -545,9 +551,14 @@ func buildBrain(w *chatWindow, session string, opts brainOptions) (*chatBrain, e fanIn = store.DependencyFanIn{} } // Ordinary leaves retain the byte-identical headless envelope, raised by - // exactly what their own fan-in measures — see gatheringGrant. Reflexes use - // the deliberately tiny rung budget and a seconds-scale watchdog. + // exactly what their own fan-in measures — see gatheringGrant. A leaf + // re-dispatched after running out is granted more than the attempt that + // ran out, or it would buy the same truncated ending again — see + // regrantAfterRunningOut; the wall below follows the larger grant because + // it is arithmetic over the same number. Reflexes use the deliberately + // tiny rung budget and a seconds-scale watchdog. turns, tokens := gatheringGrant(chatLeafTurns, chatLeafTokens, fanIn) + tokens = regrantAfterRunningOut(tokens, int(node.Attempt)) leafRoom := exec.SubharnessFor(subharness) wallLeft := remainingWall(ctx) deadline := leafRoom.DeadlineWithin(tokens, wallLeft) diff --git a/cmd/codeaf/chatv3.go b/cmd/codeaf/chatv3.go index 7e91f73e4c..ca73393a09 100644 --- a/cmd/codeaf/chatv3.go +++ b/cmd/codeaf/chatv3.go @@ -897,6 +897,15 @@ func openV3Launch(proc *v3Process, opts v3Options) (*v3Launch, error) { media = v3ImageGen(mediaSettings) } + // The Model Pool's judge reads every landing through the engine's one hook. + // It is built here, where the seats above are wired, so the answer to + // "may this process read the pool at all" is one posture for the whole + // door: a mode that forbids reading builds no hook, and a nil hook is the + // engine's own nothing. The ask is built once and a client is made from it + // per call, each billed to the judge's own seat. + taskLanded := poolJudgeHook(settings, settings.ProfileDir, workspace, + config.AutoModels, poolJudgeAsk(settings, settings.ProfileDir), time.Now) + cfg := session.Config{ Workspace: workspace, Model: chosen, @@ -905,6 +914,7 @@ func openV3Launch(proc *v3Process, opts v3Options) (*v3Launch, error) { Sources: settings.Sources, CompactEnabled: !opts.NoCompact, SessionFile: transcript, + TaskLanded: taskLanded, // The folder this conversation keeps everything in (Decision 26). It is // the zero Place for a launch opened on a flat legacy transcript, which // is what keeps that session deriving its sidecars the way it always did. diff --git a/cmd/codeaf/chatv3_crew_test.go b/cmd/codeaf/chatv3_crew_test.go index a615140e11..38c4cdcb66 100644 --- a/cmd/codeaf/chatv3_crew_test.go +++ b/cmd/codeaf/chatv3_crew_test.go @@ -122,7 +122,7 @@ func TestABrokenRowLeavesTheLastGoodCrewInPlace(t *testing.T) { t.Fatal(err) } want, _ := config.CrewModels(config.CrewMax) - if model, _ := roles.Resolve(roles.Source(source), roles.RolePlanner, "vendor/conversation"); model != "moonshotai/kimi-k3" { + if model, _ := roles.Resolve(roles.Source(source), roles.RolePlanner, "vendor/conversation"); model != want[config.ModelTierMastermind] { t.Fatalf("after the row parsed again the planner resolves to %q, want the max crew's %q", model, want[config.ModelTierMastermind]) } diff --git a/cmd/codeaf/chatv3_host.go b/cmd/codeaf/chatv3_host.go index d08ef3c490..3aa9acc79b 100644 --- a/cmd/codeaf/chatv3_host.go +++ b/cmd/codeaf/chatv3_host.go @@ -614,6 +614,12 @@ func hostOptions(fleet *engineFleet, welcome remote.Welcome, pick bool) (tui3.Op } discovery := catalog.Options{BaseURL: settings.BaseURL, APIKey: settings.APIKey, Dir: profileDir} models := catalog.LoadLazy(context.Background(), discovery) + // A tier row that says auto is answered from this catalog (config.AutoModels): + // the same non-blocking read, never a fetch, and set once at start-up. + config.AutoModels = models.ModelsNow + // The pool's index is seated beside it, read once here and refreshed in the + // background, against the same profile the catalog was read from. + wirePoolIndex(profileDir) // The refresh key in /model asks the same router THIS machine's list came // from, and refills the same shelf — the list is this laptop's list of // names on both doors (chatv3_modelshelf.go). diff --git a/cmd/codeaf/chatv3_process.go b/cmd/codeaf/chatv3_process.go index ac20da97ee..531515d47d 100644 --- a/cmd/codeaf/chatv3_process.go +++ b/cmd/codeaf/chatv3_process.go @@ -177,6 +177,10 @@ func openV3ProcessWith(door string, askKey bool) (*v3Process, error) { BaseURL: settings.BaseURL, APIKey: settings.APIKey, Dir: settings.ProfileDir, } models := catalog.LoadLazy(context.Background(), discovery) + // A tier row that says auto is answered from this catalog (config.AutoModels): + // the same non-blocking read, never a fetch, and set once at start-up. + config.AutoModels = models.ModelsNow + wirePoolIndex(settings.ProfileDir) shelf := newV3ModelShelf(models, discovery) shelf.setSources(settings.Sources) return &v3Process{ diff --git a/cmd/codeaf/context_budget_test.go b/cmd/codeaf/context_budget_test.go index 883632683f..7c7cc00348 100644 --- a/cmd/codeaf/context_budget_test.go +++ b/cmd/codeaf/context_budget_test.go @@ -208,3 +208,60 @@ func TestAFoldsGrantIsSizedForThePassItActuallyMakes(t *testing.T) { t.Fatalf("an unsized window granted %d tokens, want the process-wide %d", unknown, want) } } + +// A RE-DISPATCH IS GRANTED MORE ROOM THAN THE ATTEMPT THAT RAN OUT, or the +// identical budget buys the identical truncated ending. The runner's overrun +// settle releases a leaf that ran out back to pending with its banked turns, +// and while the grant was attempt-blind the next claim re-entered the same +// brief under the same meter and stopped at the same place — one truncation, +// paid for as many times as the round cap allowed. +func TestARedispatchIsGrantedMoreRoomThanTheAttemptThatRanOut(t *testing.T) { + // Attempt zero keeps the grant byte for byte: a leaf on its first claim + // has run out of nothing yet. + if room := regrantAfterRunningOut(chatLeafTokens, 0); room != chatLeafTokens { + t.Fatalf("a first claim was granted %d tokens, want the flat %d it has always had", + room, chatLeafTokens) + } + + // Half again per re-dispatch, compounding, and strictly more room each + // time — the growth is the whole point of sending a node round again. + previous := chatLeafTokens + for attempt := 1; attempt <= 3; attempt++ { + room := regrantAfterRunningOut(chatLeafTokens, attempt) + if room <= previous { + t.Fatalf("re-dispatch %d was granted %d tokens, no more than the %d before it", + attempt, room, previous) + } + previous = room + } + // The exact arithmetic, in the integers the grant is stated in: three + // halves, applied per attempt, never rounded up by a shortcut. + if got, want := regrantAfterRunningOut(chatLeafTokens, 2), chatLeafTokens*3/2*3/2; got != want { + t.Fatalf("attempt 2 was granted %d tokens, want the twice-grown %d", got, want) + } + + // The ceiling holds at attempts no run can reach, and the ladder stops + // there rather than stepping over it. + for _, attempt := range []int{10, 100} { + if room := regrantAfterRunningOut(chatLeafTokens, attempt); room != overrunGrantCeiling { + t.Fatalf("attempt %d was granted %d tokens, want the ceiling %d", + attempt, room, overrunGrantCeiling) + } + } + + // A leaf whose fan-in already measured more than the ceiling keeps every + // token it measured. The bound is on what the ladder adds, and a + // re-dispatch handed less room than its first attempt would be the + // identical-budget defect back by another door. + _, measured := gatheringGrant(chatLeafTurns, chatLeafTokens, + store.DependencyFanIn{Count: 8, Bytes: 8 << 20}) + if measured <= overrunGrantCeiling { + t.Fatalf("the wide join measured %d tokens, too small to ask the question", measured) + } + for _, attempt := range []int{1, 3, 100} { + if room := regrantAfterRunningOut(measured, attempt); room != measured { + t.Fatalf("a join measured at %d tokens was re-dispatched at %d on attempt %d", + measured, room, attempt) + } + } +} diff --git a/cmd/codeaf/coredone_test.go b/cmd/codeaf/coredone_test.go new file mode 100644 index 0000000000..32e3a251f4 --- /dev/null +++ b/cmd/codeaf/coredone_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "path/filepath" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/store" +) + +// THE ENVELOPE SAYS WHEN THE REQUESTED WORK WAS FIRST FOUND DONE, in seconds +// from the run's start, off the journal's own gate event — and leaves the key +// out when no gate ever said so, because a fabricated instant is worse than an +// absent one. +func TestTheEnvelopeSaysWhenTheRequestedWorkWasDone(t *testing.T) { + graph, err := store.Open(filepath.Join(t.TempDir(), "coredone.db")) + if err != nil { + t.Fatal(err) + } + defer graph.Close() + + const gated, ungated = "session-gated", "session-ungated" + spliceForErrand(t, graph, gated, []store.NodeSpec{{ID: "task-1", Brief: "add the flag"}}) + spliceForErrand(t, graph, ungated, []store.NodeSpec{{ID: "task-2", Brief: "add the other flag"}}) + if err := graph.RecordDeliveryGate("task-1", store.DeliveryGate{Pass: true}); err != nil { + t.Fatal(err) + } + _, at, ok, err := graph.DeliveryGateAnchor("task-1") + if err != nil || !ok { + t.Fatalf("the passing gate has no anchor: ok=%t err=%v", ok, err) + } + started := at.Add(-7 * time.Second) + + gatedOutcome := headlessOutcome{started: started} + priceErrand(graph, gated, 0, &gatedOutcome) + fields := envelopeFields(t, errandEnvelope(gatedOutcome)) + got, present := fields["core_done_seconds"] + if !present { + t.Fatal("a gate found the work done and the envelope says nothing about when") + } + if seconds, ok := got.(float64); !ok || seconds < 6.9 || seconds > 7.1 { + t.Fatalf("core_done_seconds = %v, want about 7", got) + } + + // And absent when no gate ever said the work was done. + ungatedOutcome := headlessOutcome{started: started} + priceErrand(graph, ungated, 0, &ungatedOutcome) + if _, present := envelopeFields(t, errandEnvelope(ungatedOutcome))["core_done_seconds"]; present { + t.Fatal("a run whose gate never found the work done reported when it was") + } +} diff --git a/cmd/codeaf/do.go b/cmd/codeaf/do.go index 787c8cd0be..6dcd8855e4 100644 --- a/cmd/codeaf/do.go +++ b/cmd/codeaf/do.go @@ -110,20 +110,29 @@ type headlessOutcome struct { Nodes int `json:"nodes"` Seconds float64 `json:"seconds"` - Settled bool `json:"settled"` + // started is when this invocation opened, and coreDoneSeconds is how long + // it took to finish the requested work: the first moment a delivery gate + // found that work done. Both are unexported — they reach a caller only + // through the envelope's `core_done_seconds`, which is the one spelling + // every reader shares. + started time.Time + coreDoneSeconds float64 + Settled bool `json:"settled"` // unfinishedTree is the finished-tree reading's own sentence on the one run // that cannot be called settled: its checks failed to collect. It stays // unexported because the sentence leaves through Deliverable, while Settled // is already the machine signal and the JSON contract needs no second key. unfinishedTree string - // Run, Calls and Rounds are what a person went to `calls.jsonl` to - // reconstruct: which run this was, how many model calls it made, and how - // many times it bought more work after looking at what it had. They are - // unexported spellings of the envelope's own keys — the receipt reaches a - // caller through [errandEnvelope] and nowhere else. - run string - calls int - rounds int + // Run, Calls, Rounds and Redispatches are what a person went to + // `calls.jsonl` to reconstruct: which run this was, how many model calls it + // made, how many times it bought more work after looking at what it had, + // and how many times it sent a node round again in place after the node ran + // out of its room. They are unexported spellings of the envelope's own keys + // — the receipt reaches a caller through [errandEnvelope] and nowhere else. + run string + calls int + rounds int + redispatches int // tokensIn and tokensOut are the token half of the bill, summed out of the // same journal read that priced the run. They are unexported because they // reach a caller only through the envelope's `tokens` field, which is the @@ -390,6 +399,15 @@ func doErrand(request doRequest) error { // The two seats, resolved before anything is opened or built, so the run // says which models it is about to use and on whose authority — and says it // even on a run that dies before it reaches a provider. + // + // The catalog is seated under the ladder first, because a tier row may say + // `auto` and that word is answered from the rows this process already holds + // (useAutoSeats). It is read keyless: a run with no key fails further down + // with a sentence about the key, and a seat that fell to the table row on + // the way there would report a model this run never meant to use. + if settings, err := config.LoadKeyless(); err == nil { + useAutoSeats(settings) + } seats := config.ResolveSeats(config.ProfileDir(), request.model, request.planModel) fmt.Fprintln(request.stderr, seats.Report()) outcome, err := errandRun(request, seats, started) @@ -631,6 +649,7 @@ func errandRun(request doRequest, seats config.Seats, started time.Time) (outcom return headlessOutcome{}, err } outcome.Seconds = time.Since(started).Seconds() + outcome.started = started // The wall is the case that made this necessary. A leaf cancelled by the // timeout journals its usage row on the way down, which is after the // watcher has returned and — until this line moved the shutdown ahead of @@ -671,6 +690,52 @@ func priceErrand(graph *store.Store, session string, openedAt int64, outcome *he // work is read from what was written down, for the reason the bill is: a // counter in this process could not see a round a resident spliced. outcome.rounds = errandRounds(graph, session) + // AND THE RE-DISPATCHES OFF IT TOO. How many times this run sent a node + // round again in place is read from the releases that handed work on, for + // the same reason: the release is the record of a re-dispatch, and a + // counter in this process could not see one a resident made. + outcome.redispatches = errandRedispatches(graph, session) + // AND WHEN THE REQUESTED WORK WAS FIRST FOUND DONE, off the same journal. + outcome.coreDoneSeconds = errandCoreDoneSeconds(graph, session, outcome.started) +} + +// errandCoreDoneSeconds is how long it took this errand to finish the requested +// work, measured from the run's start to the first delivery gate on one of its +// jobs that found the work done — a pass, or a coverage finding. It is the +// EARLIEST such moment across the errand's jobs, so a run that finished one part +// early and another late reports the first; a run whose gate never said so +// reports zero, which the envelope carries as an absent key rather than a +// fabricated instant. +func errandCoreDoneSeconds(graph *store.Store, session string, started time.Time) float64 { + if graph == nil || started.IsZero() { + return 0 + } + nodes, err := graph.SessionMemberNodes(session) + if err != nil { + return 0 + } + var earliest int64 + var at time.Time + for _, node := range nodes { + if node.Parent != store.RootID { + continue + } + seq, when, ok, err := graph.DeliveryGateAnchor(node.ID) + if err != nil || !ok { + continue + } + if earliest == 0 || seq < earliest { + earliest, at = seq, when + } + } + if earliest == 0 { + return 0 + } + seconds := at.Sub(started).Seconds() + if seconds < 0 { + return 0 + } + return seconds } // errandRounds is how many times this errand bought MORE WORK: every growth @@ -700,6 +765,31 @@ func errandRounds(graph *store.Store, session string) int { return rounds } +// errandRedispatches is how many times one of this errand's nodes was sent +// round again in place after running out of the room it was granted: every +// hand-on release its nodes journaled (store.NodeRedispatches). +// +// It asks per node rather than across the store because a release is journaled +// against the node that was re-dispatched, and the rule is the bill's and the +// rounds': a run sharing a durable store with another session must not count +// that session's re-dispatches as its own. A read that fails leaves the count +// at zero rather than at a guess. +func errandRedispatches(graph *store.Store, session string) int { + nodes, err := graph.SessionMemberNodes(session) + if err != nil { + return 0 + } + redispatches := 0 + for _, node := range nodes { + counted, err := graph.NodeRedispatches(node.ID) + if err != nil { + continue + } + redispatches += counted + } + return redispatches +} + // headlessBrain builds and returns the brain this process will run, or nothing // at all when another process is already the brain for this store. // @@ -2985,20 +3075,22 @@ func reportErrand(request doRequest, outcome headlessOutcome) error { // `exec` and `run` from publishing three different objects again. func errandEnvelope(outcome headlessOutcome) resultEnvelope { return buildResultEnvelope(runResult{ - Stop: outcome.resolvedStop(), - Answer: outcome.Deliverable, - Files: outcome.Artifacts, - Error: outcome.Error, - SpendUSD: outcome.Spend, - TokensIn: outcome.tokensIn, - TokensOut: outcome.tokensOut, - Seconds: outcome.Seconds, - Model: outcome.Model, - Steps: outcome.Nodes, - Run: outcome.run, - Calls: outcome.calls, - Rounds: outcome.rounds, - Extra: legacyErrandFields(outcome), + Stop: outcome.resolvedStop(), + Answer: outcome.Deliverable, + Files: outcome.Artifacts, + Error: outcome.Error, + SpendUSD: outcome.Spend, + TokensIn: outcome.tokensIn, + TokensOut: outcome.tokensOut, + Seconds: outcome.Seconds, + CoreDoneSeconds: outcome.coreDoneSeconds, + Model: outcome.Model, + Steps: outcome.Nodes, + Run: outcome.run, + Calls: outcome.calls, + Rounds: outcome.rounds, + Redispatches: outcome.redispatches, + Extra: legacyErrandFields(outcome), }) } diff --git a/cmd/codeaf/do_test.go b/cmd/codeaf/do_test.go index 852ae14f68..73595abce3 100644 --- a/cmd/codeaf/do_test.go +++ b/cmd/codeaf/do_test.go @@ -2330,3 +2330,70 @@ func TestAnErrandCountsTheRoundsItsOwnJobsBought(t *testing.T) { "a neighbour's rounds as its own", got) } } + +// TestAnErrandCountsTheRedispatchesItsOwnNodesBought is the figure beside the +// rounds, and the reason it comes off the journal rather than a counter. +// +// `do --json` said how many times a run went back for MORE but never how many +// times it sent a node round again IN PLACE — a leaf that ran out of its room +// going back on the queue to be carried on from, which is the other half of +// what a bill of re-runs is made of. +// +// IT IS COUNTED PER NODE AND NOT ACROSS THE STORE, for the reason the rounds +// are: a run sharing a durable store with another session must not report that +// session's re-dispatches as its own. +func TestAnErrandCountsTheRedispatchesItsOwnNodesBought(t *testing.T) { + path := filepath.Join(t.TempDir(), "redispatches.db") + graph, err := store.Open(path) + if err != nil { + t.Fatal(err) + } + defer graph.Close() + + const mine, theirs = "session-mine", "session-theirs" + for _, job := range []struct{ id, session string }{ + {"job-a", mine}, {"job-b", mine}, {"job-c", theirs}, + } { + if err := graph.Splice(store.RootID, store.Subtree{Nodes: []store.NodeSpec{{ + ID: job.id, Brief: "a job", Stage: 1, + }}}, store.Provenance{Origin: store.OriginUser, Intent: "a job", SessionID: job.session}); err != nil { + t.Fatal(err) + } + } + // Two hand-ons on one of this errand's nodes, one on another, a hand-back + // that carried nothing on — which is a claim taken off a worker, not a + // re-dispatch — and one on a node belonging to somebody else entirely. + for _, handed := range []struct { + node string + reason string + recorded int + }{ + {"job-a", "it was still working when it ran out of its token budget", 3}, + {"job-a", "it was still working when it ran out of its token budget", 3}, + {"job-b", "it was still working when it ran out of its token budget", 2}, + {"job-b", "no sign of life for the claim", 0}, + {"job-c", "it was still working when it ran out of its token budget", 1}, + } { + claim, ok, err := graph.Claim(handed.node, "chat-runner") + if err != nil || !ok { + t.Fatalf("claim %s: %v", handed.node, err) + } + if handed.recorded > 0 { + if err := graph.ReleaseWithRecord(claim, handed.reason, handed.recorded); err != nil { + t.Fatal(err) + } + continue + } + if err := graph.ReleaseWithReason(claim, handed.reason); err != nil { + t.Fatal(err) + } + } + + if got := errandRedispatches(graph, mine); got != 3 { + t.Errorf("this errand's nodes were sent round again three times between them and it reports %d", got) + } + if got := errandRedispatches(graph, theirs); got != 1 { + t.Errorf("the other session re-dispatched once and it reports %d — a run may not "+ + "count a neighbour's re-dispatches as its own", got) + } +} diff --git a/cmd/codeaf/envelope.go b/cmd/codeaf/envelope.go index f191944547..09ea4c16c3 100644 --- a/cmd/codeaf/envelope.go +++ b/cmd/codeaf/envelope.go @@ -294,6 +294,13 @@ type resultEnvelope struct { Tokens envelopeTokens `json:"tokens"` // Seconds is how long it took, wall clock. Seconds float64 `json:"seconds"` + // CoreDoneSeconds is when the requested work was FIRST found done, in + // seconds from the run's start: the first delivery gate that passed or left + // only a coverage finding — the fix committed, the checks for it still to + // come. It is omitted when no gate ever said so, and it is the figure a + // person reads to see how much of the run was the work and how much came + // after it. + CoreDoneSeconds float64 `json:"core_done_seconds,omitempty"` // Model is the model the work ran on, as the seat ladder resolved it. Model string `json:"model"` // Steps is how many pieces of work ran: `do`'s nodes, `exec`'s turns. A @@ -333,6 +340,19 @@ type resultEnvelope struct { // both because a caller reaching for a key that vanished is a caller // crashing. Rounds int `json:"rounds"` + // Redispatches is how many times one of this run's nodes was re-dispatched + // IN PLACE after running out of the room it was granted — sent round again + // to carry on from what it had banked, rather than grown around. It is + // counted off the journal the way Rounds is (store.NodeRedispatches), for + // the same reason: a run sharing a durable store with another session must + // not count that session's re-dispatches as its own. + // + // It is the one contract field that is ABSENT rather than zero when it never + // happened, which is what `omitempty` buys and why it wears it where Rounds + // does not: a caller of this one is asking whether the run had to send + // anything round again, and "never" is the key not being there. `exec` and + // a saved program never set it, so neither ever carries it. + Redispatches int `json:"redispatches,omitempty"` // extra is what one verb carries beyond the contract, and it is two things: // the OLD field names, kept readable for one release so that a tool written @@ -392,11 +412,16 @@ type runResult struct { TokensIn int TokensOut int Seconds float64 - Model string - Steps int - Run string - Calls int - Rounds int + // CoreDoneSeconds is `do`'s fact: when its gate first found the requested + // work done, read off the journal. `exec` and `run` have no delivery gate + // and leave it zero, which omits the key. + CoreDoneSeconds float64 + Model string + Steps int + Run string + Calls int + Rounds int + Redispatches int // Extra is this verb's own fields: its old spellings, and whatever it knows // that the contract has no room for. Nil for a verb with neither. Extra map[string]any @@ -421,20 +446,22 @@ func buildResultEnvelope(result runResult) resultEnvelope { files = []string{} } return resultEnvelope{ - OK: exitFor(stop) == exitDone, - Stop: stop, - Answer: result.Answer, - Files: files, - Error: result.Error, - SpendUSD: result.SpendUSD, - Tokens: envelopeTokens{In: result.TokensIn, Out: result.TokensOut}, - Seconds: result.Seconds, - Model: result.Model, - Steps: result.Steps, - Run: result.Run, - Calls: result.Calls, - Rounds: result.Rounds, - extra: result.Extra, + OK: exitFor(stop) == exitDone, + Stop: stop, + Answer: result.Answer, + Files: files, + Error: result.Error, + SpendUSD: result.SpendUSD, + Tokens: envelopeTokens{In: result.TokensIn, Out: result.TokensOut}, + Seconds: result.Seconds, + CoreDoneSeconds: result.CoreDoneSeconds, + Model: result.Model, + Steps: result.Steps, + Run: result.Run, + Calls: result.Calls, + Rounds: result.Rounds, + Redispatches: result.Redispatches, + extra: result.Extra, } } diff --git a/cmd/codeaf/envelope_test.go b/cmd/codeaf/envelope_test.go index 531137b2cb..de4bd0d3f8 100644 --- a/cmd/codeaf/envelope_test.go +++ b/cmd/codeaf/envelope_test.go @@ -399,4 +399,19 @@ func TestTheEnvelopeNamesItsRunAndCountsItsCallsAndRounds(t *testing.T) { if fields["rounds"] != float64(8) { t.Errorf("rounds = %v, want the 8 rounds of work it bought", fields["rounds"]) } + // `redispatches` carries its count the same way, and is the one key that is + // absent rather than zero when it never happened — a caller of it is asking + // whether the run had to send anything round again, and "never" is the key + // not being there. + if _, present := fields["redispatches"]; present { + t.Errorf("redispatches = %v on a run that re-dispatched nothing; the key belongs "+ + "to the runs it happened to", fields["redispatches"]) + } + fields = envelopeFields(t, buildResultEnvelope(runResult{ + Run: "0123456789abcdef", Calls: 422, Rounds: 8, Redispatches: 1, + })) + if fields["redispatches"] != float64(1) { + t.Errorf("redispatches = %v, want the one node the run sent round again in place", + fields["redispatches"]) + } } diff --git a/cmd/codeaf/exec.go b/cmd/codeaf/exec.go index f6b92cb6a2..57e0765a1a 100644 --- a/cmd/codeaf/exec.go +++ b/cmd/codeaf/exec.go @@ -113,6 +113,7 @@ func runExec(args []string) error { // that took the flag and then resolved it differently from every other door // would be the parity it claims in name only. Only the work seat is printed, // because only the work seat runs anything. + useAutoSeats(settings) seats := config.ResolveSeats(settings.ProfileDir, *model, *planModel) applySeats(&settings, seats) fmt.Fprintln(os.Stderr, seats.Work.Report()) diff --git a/cmd/codeaf/main.go b/cmd/codeaf/main.go index 3b448b3404..4df6371d29 100644 --- a/cmd/codeaf/main.go +++ b/cmd/codeaf/main.go @@ -216,6 +216,8 @@ func run() error { os.Args[2:], func(args []string) error { return runShow("plan show", args) }) case "models": return runModels(os.Args[2:]) + case "pool": + return runPool(os.Args[2:]) case "notebook": return runNotebook(os.Args[2:]) case "collections": @@ -373,10 +375,11 @@ Look at what happened — read-only, no key, nothing spent codeaf logs [--tail 40] [--follow] [--path] [--json] [--run id] [--call id] [--tag t] [--model m] [--node n] [--body id] every model call codeaf made — what was asked, which lane answered, what - came back; --json prints the rows as on disk, --body one call's bodies. - The filters are exact and combine; CODEAF_CALL_LOG=off is off + came back; --json as on disk, --body one call, CODEAF_CALL_LOG=off is off codeaf models [--refresh] the models this machine will use, and what each has been measured at + codeaf pool [show|status|verify] [--json] [--key key] + the Model Pool: what is resolved and cached; verify fetches with a key codeaf doctor [--db path] is this install healthy, and where does it keep things codeaf manual @@ -417,8 +420,7 @@ Housekeeping — changes state on disk or on the network Plan work by hand — a plan you can read, edit and diff codeaf plan new "" [--out plan.json] [--dir dir] [--json] - [--instructions] [--passes auto|off|N] [--model slug] - [--plan-model slug] + [--instructions] [--passes auto|off|N] [--plan-model slug] codeaf plan show codeaf plan revise "" [--done 1,2,3] [--out plan.json] [--model slug] [--plan-model slug] @@ -656,6 +658,7 @@ func runPlanNew(name string, args []string) error { if err != nil { return err } + useAutoSeats(settings) seats := config.ResolveSeats(settings.ProfileDir, *model, *planModel) applySeats(&settings, seats) workClient, err := settings.Client() @@ -835,6 +838,7 @@ func runRevise(name string, args []string) error { if err != nil { return err } + useAutoSeats(settings) seats := config.ResolveSeats(settings.ProfileDir, *model, *planModel) applySeats(&settings, seats) workClient, err := settings.Client() diff --git a/cmd/codeaf/pool.go b/cmd/codeaf/pool.go new file mode 100644 index 0000000000..65bb36ef90 --- /dev/null +++ b/cmd/codeaf/pool.go @@ -0,0 +1,674 @@ +// The Model Pool from the command line. +// +// `codeaf pool` answers what the pool is on this machine: the mode and the +// addresses in force with the word saying where each came from, the index +// cached from the last good read, and — under status — what is waiting to be +// sent and whether the relay and the mirror answered. It spends nothing, and +// reaches the network only under `status` and `verify`, each of which fetches +// the index: `status` to say whether an address answers, `verify` to check a +// fresh one's signature. THE KEY STANDS IN FRONT OF THE FETCH: a signature +// nobody can check is a fetch nobody should make, so a build left with no key +// in hand refuses verify at the door rather than downloading bytes it cannot +// vouch for. +package main + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/pool/index" + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" + "github.com/Agent-Field/codeaf/internal/pool/pull" + "github.com/Agent-Field/codeaf/internal/pool/record" + "github.com/Agent-Field/codeaf/internal/tui2/reltime" +) + +// poolPublicKeys are the ed25519 public keys a fetched index's signature is +// checked under when no key is stored beside the pool row. The list carries +// the key the index signer publishes (relay/wrangler.toml's POOL_PUBLIC_KEY), +// decoded once here from its base64 literal: a build whose literal could not +// decode would verify nothing, so the test beside the verb pins the length. +var poolPublicKeys = mustPoolKey("WOAo+g/oKxAV9vVqv2Q14w1TyyyiouwFO2fC0zgcps0=") + +func mustPoolKey(encoded string) []ed25519.PublicKey { + raw, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || len(raw) != ed25519.PublicKeySize { + panic(fmt.Sprintf("pool: the built-in public key does not decode: %q", encoded)) + } + return []ed25519.PublicKey{ed25519.PublicKey(raw)} +} + +// poolTrustedKeys resolves the keys a fetched index is checked under: the +// stored key when one is set, AND ONLY IT — the row is the one word the +// install trusts, so a word that does not decode is a key nobody can vouch +// for, and the answer is no keys rather than a fall back to the built-in one +// a fetch under would verify nothing with. Nothing stored: the key the build +// carries. Whether the stored word was there at all or there and broken is +// [poolTrustedKeysErr]'s to say. +func poolTrustedKeys(cfg poolcfg.Config) []ed25519.PublicKey { + keys, _ := poolTrustedKeysErr(cfg) + return keys +} + +// poolTrustedKeysErr is the same answer with the stored word's health beside +// it: a key set but not decodable is not the ordinary nothing — the install +// holds a word it cannot vouch for anything under, and the verb that would +// fetch is refused with the row's name rather than left silent. +func poolTrustedKeysErr(cfg poolcfg.Config) ([]ed25519.PublicKey, error) { + word := strings.TrimSpace(cfg.PublicKey) + if word == "" { + return poolPublicKeys, nil + } + raw, err := base64.StdEncoding.DecodeString(word) + if err != nil || len(raw) != ed25519.PublicKeySize { + return nil, errPoolKeyDoesNotDecode + } + return []ed25519.PublicKey{ed25519.PublicKey(raw)}, nil +} + +// errPoolKeyDoesNotDecode marks a stored word that is not a key; the sentence +// a person reads is [poolKeyReason]'s, which names the word where it was set. +var errPoolKeyDoesNotDecode = errors.New("the stored public key does not decode") + +// poolKeyReason is the one line a key that does not decode is refused by: the +// row when the word was stored, the environment pin when poolcfg resolved it +// from there — the remedy belongs to whichever word is in force. +func poolKeyReason(cfg poolcfg.Config) string { + if cfg.Source.PublicKey == "env" { + return "CODEAF_MODEL_POOL_PUBLIC_KEY does not decode (set it to the base64 Ed25519 public key of your relay, or clear it)" + } + return "models.pool.public_key does not decode (set it to the base64 Ed25519 public key of your relay, or clear it)" +} + +// poolKeys is --key's value: a base64 ed25519 public key, repeatable, each +// decoded the moment it is typed rather than carried as text and decoded at +// the fetch. A key that does not decode is refused where it was typed. +type poolKeys []ed25519.PublicKey + +func (k *poolKeys) String() string { + if len(*k) == 0 { + return "" + } + return fmt.Sprintf("%d key(s)", len(*k)) +} + +func (k *poolKeys) Set(word string) error { + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(word)) + if err != nil { + return fmt.Errorf("--key: %q is not base64", word) + } + if len(raw) != ed25519.PublicKeySize { + return fmt.Errorf("--key: %d bytes, want an ed25519 public key's %d", len(raw), ed25519.PublicKeySize) + } + *k = append(*k, ed25519.PublicKey(raw)) + return nil +} + +func runPool(args []string) error { + return runPoolWith(args, os.Stdout, config.ProfileDir(), time.Now, os.LookupEnv) +} + +// runPoolWith is the verb with its outside readings injectable: where the +// profile is, what time it is, and the environment the resolver reads — the +// last so a test can pin the pool's names without touching the process. The +// stored word is resolved once, here, and every form below answers from that +// one Config. The pool's files live in the profile, and the profile is +// resolved the way every other file under it is ([config.ProfilePath]) — +// emptiness is the state root's own profile, never a directory called "pool" +// beside wherever the command happened to run. +func runPoolWith(args []string, output io.Writer, profileDir string, now func() time.Time, lookup func(string) (string, bool)) error { + cfg := poolcfg.Resolve(config.ModelPoolSettingAt(profileDir), config.ModelPoolPublicKeySettingAt(profileDir), lookup) + poolDir := config.ProfilePath(profileDir, "pool") + if len(args) == 0 { + args = []string{"show"} + } + switch args[0] { + case "show": + return showPool(args[1:], output, poolDir, cfg, now) + case "status": + return statusPool(args[1:], output, poolDir, cfg, now) + case "verify": + return verifyPool(args[1:], output, poolDir, cfg, now) + default: + // `codeaf pool --help` reaches here rather than a flag set, because + // the reading form parses nothing at all (usage.go). + if askedForHelp(args) { + return commandHelp("pool") + } + return fmt.Errorf("usage: codeaf pool [show|status|verify] [--json]") + } +} + +// showPool is the reading form. statusPool is the same answer with the outbox, +// the two doors the mode opens, and the relay's own answer added to it — the +// last is why status, alone among the reading forms, asks the network. +func showPool(args []string, output io.Writer, poolDir string, cfg poolcfg.Config, now func() time.Time) error { + flags := commandFlags("pool show") + asJSON := flags.Bool("json", false, "print the answer as one JSON object") + if err := parseCommandFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("usage: codeaf pool show [--json]") + } + return printPool(output, poolDir, cfg, now(), *asJSON, false, nil) +} + +func statusPool(args []string, output io.Writer, poolDir string, cfg poolcfg.Config, now func() time.Time) error { + flags := commandFlags("pool status") + asJSON := flags.Bool("json", false, "print the answer as one JSON object") + var keys poolKeys + flags.Var(&keys, "key", "an ed25519 public key, base64; repeatable") + if err := parseCommandFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("usage: codeaf pool status [--json] [--key key]") + } + return printPool(output, poolDir, cfg, now(), *asJSON, true, keys) +} + +// printPool is the reading form's whole answer. The config first — every value +// beside the word saying where it came from, one of default, setting, env or +// ci — then the cached index with its age, then the install's own sheet, +// then, for status, the outbox and the two doors the mode opens. +func printPool(output io.Writer, poolDir string, cfg poolcfg.Config, now time.Time, asJSON, withStatus bool, keys []ed25519.PublicKey) error { + var cached *index.Index + // The cache is read the way show reads everything else, as an answer and + // not as an argument: a document that does not parse is not there yet, + // and the line below says so. The signature and the puller's version mark + // are verify's business; show reports what a person has. + if doc, err := os.ReadFile(filepath.Join(poolDir, "doc.json")); err == nil { + if parsed, err := index.Parse(doc); err == nil { + cached = parsed + } + } + if asJSON { + return printPoolJSON(output, poolDir, cfg, cached, now, withStatus, keys) + } + for _, line := range []string{ + fmt.Sprintf("mode %s · %s", cfg.Mode, cfg.Source.Mode), + fmt.Sprintf("relay %s · %s", cfg.RelayURL, cfg.Source.RelayURL), + fmt.Sprintf("index %s · %s", cfg.IndexURL, cfg.Source.IndexURL), + fmt.Sprintf("mirror %s · %s", orNowhere(cfg.MirrorURL), cfg.Source.MirrorURL), + fmt.Sprintf("submit %s · %s", orNowhere(cfg.SubmitURL), cfg.Source.SubmitURL), + fmt.Sprintf("ttl %s · %s", reltime.Elapsed(cfg.TTL), cfg.Source.TTL), + } { + if _, err := fmt.Fprintln(output, line); err != nil { + return err + } + } + if cached == nil { + // A nothing is said in a sentence, the way an empty cache is: + // silence and a bare header both read as a command that broke. And the + // index the build carries is named beside it, so a person knows there + // are numbers before any fetch: the seed is what a pick reads until a + // fresher signed one is cached. + line := "no index cached yet" + if seed, err := index.SeedIndex(); err == nil { + line = fmt.Sprintf("no index cached yet · built-in seed of %s, %s", seed.Generated().Format("2006-01-02"), countWord(indexCellCount(seed), "cell", "cells")) + } + if _, err := fmt.Fprintln(output, line); err != nil { + return err + } + } else { + generated := cached.Generated() + if _, err := fmt.Fprintf(output, + "index · generated %s · %s old · schema %d · %s · %s · %s · min installs %d\n", + generated.Format("2006-01-02"), reltime.Elapsed(now.Sub(generated)), + cached.Schema(), countWord(len(cached.Metrics()), "metric", "metrics"), + countWord(len(cached.Judges()), "judge", "judges"), countWord(indexCellCount(cached), "cell", "cells"), cached.MinInstalls()); err != nil { + return err + } + } + // The install's own sheet is said the way every other nothing here is + // said: a sheet that holds no cell yet is `none`, and one that holds some + // is counted with its noun. + own := ownSheetSummary(poolDir) + if own.Cells == 0 { + if _, err := fmt.Fprintln(output, "own sheet: none"); err != nil { + return err + } + } else if _, err := fmt.Fprintf(output, "own sheet: %s, %s\n", + countWord(own.Cells, "cell", "cells"), countWord(own.Observations, "observation", "observations")); err != nil { + return err + } + // The last judge's record is read the way every other nothing here is + // read: a missing or malformed file is no judge yet, said in a sentence. + last := readJudgeLast(poolDir) + if withStatus { + if _, err := fmt.Fprintln(output, judgeLastLine(last)); err != nil { + return err + } + relay, mirror := probePool(poolDir, cfg, now, keys) + if _, err := fmt.Fprintln(output, relayStatusLine(cfg, relay, mirror, cached)); err != nil { + return err + } + _, err := fmt.Fprintf(output, "pending %d · can send %s · can read %s\n", + pendingRows(poolDir), yesNo(cfg.CanSend()), yesNo(cfg.CanRead())) + return err + } + return nil +} + +// poolProbeBudget is what status spends asking one address. It is short on +// purpose: the line is a reading a person waits for, and an address that takes +// longer than this to answer has not answered. +const poolProbeBudget = 3 * time.Second + +// probeSummary is one address's answer under status: whether it answered, the +// index version it served when it did, and — when it did not — the one-line +// reason. fromCache says the document was the copy already on disk, which the +// relay line words differently; it is not part of the JSON shape. +type probeSummary struct { + Reachable bool `json:"reachable"` + Version int64 `json:"version"` + Reason string `json:"reason"` + fromCache bool +} + +// probePool asks the relay for the index, and the mirror when the relay does +// not answer, under a pull that always goes to the network — TTL 0 — inside a +// three-second budget. It never fails a reading form: an address that does not +// answer is what the line is for, and status exits 0 either way. A mode that +// forbids reading asks nothing at all. +func probePool(poolDir string, cfg poolcfg.Config, now time.Time, keys []ed25519.PublicKey) (probeSummary, probeSummary) { + if !cfg.CanRead() { + off := probeSummary{Reason: "not read (model_pool off)"} + return off, off + } + // --key REPLACES the resolved keys: what the person typed is the whole + // list a probe is checked under, so a key nobody signed with can prove a + // document does not verify under it. Nothing typed: the stored word when + // it decodes, else the key the build carries — and a word that does not + // decode is the reason itself, with no address asked: a signature nobody + // can check is a fetch nobody should make. + trusted := keys + if len(trusted) == 0 { + resolved, keyErr := poolTrustedKeysErr(cfg) + if keyErr != nil { + broken := probeSummary{Reason: poolKeyReason(cfg)} + return broken, broken + } + trusted = resolved + } + clock := func() time.Time { return now } + relay := probeAddress(cfg.IndexURL, poolDir, trusted, clock) + if relay.Reachable { + return relay, probeSummary{} + } + if cfg.MirrorURL == "" { + return relay, probeSummary{Reason: "not configured"} + } + return relay, probeAddress(cfg.MirrorURL, poolDir, trusted, clock) +} + +// probeAddress pulls one address the way status asks it: a fresh fetch — TTL 0 +// so a young cache is not what answers — inside the probe budget, its failure +// turned into the one-line reason. +func probeAddress(url, poolDir string, keys []ed25519.PublicKey, now func() time.Time) probeSummary { + puller := &pull.Puller{ + URL: url, + Keys: keys, + CacheDir: poolDir, + TTL: 0, + Budget: poolProbeBudget, + Now: now, + } + result, err := puller.Pull(context.Background()) + if err != nil { + return probeSummary{Reason: oneLine(err.Error())} + } + return probeSummary{Reachable: true, Version: result.Version, fromCache: result.FromCache} +} + +// relayStatusLine is the one line status says about the addresses: whether the +// relay answered, the mirror beside it when the relay did not, and — when +// neither did — what the reading fell back to. A mode that forbids reading +// asks nothing and says that instead. +func relayStatusLine(cfg poolcfg.Config, relay, mirror probeSummary, cached *index.Index) string { + if !cfg.CanRead() { + return "relay: not read (model_pool off)" + } + if relay.Reachable { + if relay.fromCache { + return fmt.Sprintf("relay: reachable · index unchanged, version %d", relay.Version) + } + return fmt.Sprintf("relay: reachable · index version %d", relay.Version) + } + if mirror.Reachable { + return fmt.Sprintf("relay: unreachable (%s) · mirror: reachable · index version %d", relay.Reason, mirror.Version) + } + where := "built-in seed" + if cached != nil { + where = "cache" + } + return fmt.Sprintf("relay: unreachable (%s) · mirror: unreachable (%s) · reading %s", relay.Reason, mirror.Reason, where) +} + +// oneLine is a reason said on one line: any newline becomes a space, because +// the relay line holds the reason in parentheses beside the next word. +func oneLine(reason string) string { + return strings.ReplaceAll(strings.TrimSpace(reason), "\n", " ") +} + +// ownSummary is the install's own sheet as the reading forms carry it: the +// cells the picker reads beside the index, and the observations behind them. +type ownSummary struct { + Cells int `json:"cells"` + Observations int `json:"observations"` +} + +// ownSheetSummary counts what the install's own sheet holds for the picker: +// the role_quality cells recorded with no dim labels and the observations +// behind them. A sheet that is missing, or one that does not parse, reads as +// none — the reading form reports what a person has. +func ownSheetSummary(poolDir string) ownSummary { + sheet, err := record.LoadSheet(record.OwnSheetPath(poolDir)) + if err != nil { + return ownSummary{} + } + var summary ownSummary + for _, cell := range record.Cells(sheet) { + summary.Cells++ + summary.Observations += cell.N + } + return summary +} + +// poolAnswer is the --json shape of the reading forms: the config flat, the +// cached index under index or null. The three status fields are pointers so a +// show carries none of them — a field a form does not answer reads as +// not-asked rather than as zero. +type poolAnswer struct { + Mode string `json:"mode"` + ModeSource string `json:"mode_source"` + RelayURL string `json:"relay_url"` + IndexURL string `json:"index_url"` + MirrorURL string `json:"mirror_url"` + SubmitURL string `json:"submit_url"` + TTLSeconds int `json:"ttl_seconds"` + Pending *int `json:"pending,omitempty"` + CanSend *bool `json:"can_send,omitempty"` + CanRead *bool `json:"can_read,omitempty"` + Relay *probeSummary `json:"relay,omitempty"` + Mirror *probeSummary `json:"mirror,omitempty"` + LastJudge *judgeLast `json:"last_judge"` + Index *indexSummary `json:"index"` + Own ownSummary `json:"own"` +} + +// indexSummary is the cached index as the reading forms carry it: the +// document's own day, its age against the asking clock, and the counts a +// person checks before trusting it. Source says where the document came from — +// "cache" for the one under the profile, "seed" for the one the build carries +// when there is no cache. +type indexSummary struct { + Generated string `json:"generated"` + AgeSeconds int `json:"age_seconds"` + Schema int `json:"schema"` + Metrics int `json:"metrics"` + Judges int `json:"judges"` + Cells int `json:"cells"` + MinInstalls int `json:"min_installs"` + Source string `json:"source"` +} + +func printPoolJSON(output io.Writer, poolDir string, cfg poolcfg.Config, cached *index.Index, now time.Time, withStatus bool, keys []ed25519.PublicKey) error { + answer := poolAnswer{ + Mode: cfg.Mode.String(), + ModeSource: cfg.Source.Mode, + RelayURL: cfg.RelayURL, + IndexURL: cfg.IndexURL, + MirrorURL: cfg.MirrorURL, + SubmitURL: cfg.SubmitURL, + TTLSeconds: int(cfg.TTL / time.Second), + } + // A cached document is reported as itself; with no cache the build's seed + // stands in, so a script reading `index` sees the index a pick would read + // and the source field says which one it was. + held, source := cached, "cache" + if held == nil { + if seed, err := index.SeedIndex(); err == nil { + held, source = seed, "seed" + } + } + if held != nil { + generated := held.Generated() + answer.Index = &indexSummary{ + Generated: generated.Format("2006-01-02"), + AgeSeconds: int(now.Sub(generated) / time.Second), + Schema: held.Schema(), + Metrics: len(held.Metrics()), + Judges: len(held.Judges()), + Cells: indexCellCount(held), + MinInstalls: held.MinInstalls(), + Source: source, + } + } + if withStatus { + pending := pendingRows(poolDir) + send, read := cfg.CanSend(), cfg.CanRead() + answer.Pending = &pending + answer.CanSend = &send + answer.CanRead = &read + relay, mirror := probePool(poolDir, cfg, now, keys) + answer.Relay = &relay + answer.Mirror = &mirror + } + answer.Own = ownSheetSummary(poolDir) + // The record is null when there is none, so a script can tell a judge + // that has not run from one that failed. + if last := readJudgeLast(poolDir); last != nil { + answer.LastJudge = last + } + encoded, err := json.Marshal(answer) + if err != nil { + return err + } + _, err = fmt.Fprintf(output, "%s\n", encoded) + return err +} + +// verifyPool is the one form that reaches the network: fetch the index now — +// TTL 0, the one door that means now — and check its signature. It refuses at +// the door when the mode is off and when no key is in hand; a failed fetch +// says its error and leaves on exit 1 through the one door every failure +// leaves by. +func verifyPool(args []string, output io.Writer, poolDir string, cfg poolcfg.Config, now func() time.Time) error { + flags := commandFlags("pool verify") + asJSON := flags.Bool("json", false, "print the result as one JSON object") + var keys poolKeys + flags.Var(&keys, "key", "an ed25519 public key, base64; repeatable") + if err := parseCommandFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("usage: codeaf pool verify [--json] [--key key]") + } + // OFF IS A REFUSAL AT THE DOOR, and so is a build with no key: neither + // fetches, and the sentence is the answer, on stdout where the answer + // goes (streams.go). + // + // THE RUNG IS EXIT 2 and not exit 1: nothing was attempted, but the + // question asked of this build does not stand, and the remedy is on the + // line above the exit. verify's third refusal — a fetch or a signature + // that fails — is exit 1, in the error main() says in front of it. + // + // OFF IS A REFUSAL AT THE DOOR, and so is a key nobody can vouch for: + // neither fetches, and the sentence is the answer, on stdout where the + // answer goes (streams.go). + // + // THE RUNG IS EXIT 2 and not exit 1: nothing was attempted, but the + // question asked of this build does not stand, and the remedy is on the + // line above the exit. verify's third refusal — a fetch or a signature + // that fails — is exit 1, in the error main() says in front of it. + if cfg.Mode == poolcfg.Off { + if _, err := fmt.Fprintln(output, "the Model Pool is off in settings"); err != nil { + return err + } + return exitIncomplete + } + // --key REPLACES the resolved key: keys the person typed are the whole + // list, so a key nobody signed with can prove a document does not verify + // under it. Nothing typed: the stored word when it decodes, else the key + // the build carries — and a word that does not decode is refused by the + // name of where it was set, before anything is fetched. Only a build + // that shipped no key at all, with nothing stored, reaches the line + // after it. + trusted := []ed25519.PublicKey(keys) + if len(trusted) == 0 { + resolved, keyErr := poolTrustedKeysErr(cfg) + if keyErr != nil { + if _, err := fmt.Fprintln(output, poolKeyReason(cfg)); err != nil { + return err + } + return exitIncomplete + } + trusted = resolved + } + if len(trusted) == 0 { + if _, err := fmt.Fprintln(output, "no public key built into this build; pass --key"); err != nil { + return err + } + return exitIncomplete + } + puller := &pull.Puller{ + URL: cfg.IndexURL, + Keys: trusted, + CacheDir: poolDir, + TTL: 0, + Budget: pull.DefaultBudget, + Now: now, + } + result, err := puller.Pull(context.Background()) + if err != nil { + return err + } + held, err := index.Parse(result.Doc) + if err != nil { + return fmt.Errorf("verify: %w", err) + } + generated := held.Generated().Format("2006-01-02") + if *asJSON { + encoded, err := json.Marshal(struct { + Verified bool `json:"verified"` + Version int64 `json:"version"` + Generated string `json:"generated"` + Metrics int `json:"metrics"` + }{true, result.Version, generated, len(held.Metrics())}) + if err != nil { + return err + } + _, err = fmt.Fprintf(output, "%s\n", encoded) + return err + } + _, err = fmt.Fprintf(output, "signature good: version %d, generated %s, %d metrics\n", + result.Version, generated, len(held.Metrics())) + return err +} + +// readJudgeLast reads what the hook left about the last landing it judged: +// the judge that answered and the seats it scored, or the candidates tried +// and why none of them did. A file that is missing, or one that does not +// parse, reads as none yet — the reading form reports what a person has. +func readJudgeLast(poolDir string) *judgeLast { + data, err := os.ReadFile(filepath.Join(poolDir, "judge-last.json")) + if err != nil { + return nil + } + var last judgeLast + if json.Unmarshal(data, &last) != nil { + return nil + } + return &last +} + +// judgeLastLine is the one line status says about the last judge: which model +// answered and which seats it scored, or — when none did — how many were +// asked, the first of them, and the one-line reason the last one failed. The +// moment is the record's own, said in local hours and minutes. +func judgeLastLine(last *judgeLast) string { + if last == nil { + return "last judge: none yet" + } + at := last.At.Format("15:04") + if last.Judge != "" { + return fmt.Sprintf("last judge: %s · %s · scored %s", at, last.Judge, strings.Join(last.Scored, ", ")) + } + asked := "" + if len(last.Tried) > 0 { + asked = last.Tried[0] + if len(last.Tried) > 1 { + asked += ", …" + } + } + return fmt.Sprintf("last judge: %s · failed after %d candidates (%s) · %s", at, len(last.Tried), asked, last.Reason) +} + +// pendingRows counts what the outbox is holding. It reads the file by count +// and not by opening it, because [outbox.Open] CREATES the file when it is +// not there and a reading form must not write. +func pendingRows(poolDir string) int { + path := filepath.Join(poolDir, "outbox.jsonl") + if _, err := os.Stat(path); err != nil { + return 0 + } + box, err := outbox.Open(path) + if err != nil { + return 0 + } + defer box.Close() + return len(box.Pending()) +} + +// orNowhere is an empty submit address said rather than printed empty: the +// pin holding nothing means send nowhere, and an empty column says nothing. +func orNowhere(address string) string { + if address == "" { + return "nowhere" + } + return address +} + +// indexCellCount is the held document's cells summed over its metrics: what +// the picker can read, since a cell below min installs never parses in. +func indexCellCount(held *index.Index) int { + total := 0 + for _, metric := range held.Metrics() { + total += len(held.Cells(metric)) + } + return total +} + +// countWord is a count with its noun: one metric, three metrics, no judges. +// A zero is said rather than printed bare, for the reason every other +// nothing here is said. +func countWord(n int, one, many string) string { + switch n { + case 0: + return "no " + many + case 1: + return "1 " + one + default: + return fmt.Sprintf("%d %s", n, many) + } +} + +func yesNo(answer bool) string { + if answer { + return "yes" + } + return "no" +} diff --git a/cmd/codeaf/pool_test.go b/cmd/codeaf/pool_test.go new file mode 100644 index 0000000000..e369c05dfb --- /dev/null +++ b/cmd/codeaf/pool_test.go @@ -0,0 +1,1356 @@ +package main + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/crewpick" + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/pool/index" + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" + "github.com/Agent-Field/codeaf/internal/pool/record" + "github.com/Agent-Field/codeaf/internal/pool/tally" +) + +// poolClock is a stopped now, so an index's age is a fact rather than a race +// against the test runner. +func poolClock(t *testing.T) func() time.Time { + t.Helper() + moment, err := time.Parse(time.RFC3339, "2026-09-18T00:00:00Z") + if err != nil { + t.Fatal(err) + } + return func() time.Time { return moment } +} + +// noEnv is an environment in which nothing is set, handed in the way the verb +// reads the world, so a test's answer cannot depend on the machine's shell. +func noEnv(string) (string, bool) { + return "", false +} + +// oneEnv is an environment with one name set — the injected stand-in for +// `CODEAF_MODEL_POOL=off codeaf pool show`. +func oneEnv(name, value string) func(string) (string, bool) { + return func(asked string) (string, bool) { + if asked == name { + return value, true + } + return "", false + } +} + +// poolDoc is a document the reader accepts whole: one metric with dims, one +// cell that clears its min installs, one judge. The numbers name no model; +// it is a fixture, and the field it is read for is its shape. +func poolDoc() []byte { + return []byte(`{ + "version": 7, + "schema": 1, + "generated": "2026-09-10", + "min_installs": 1, + "judges": ["z-ai/glm-5.3"], + "metrics": {"role_rating": {"kind": "gaussian", "dims": ["role", "model"]}}, + "cells": [ + {"metric": "role_rating", "role": "planner", "model": "z-ai/glm-5.3", "mean": 1312, "sd": 18, "n": 9} + ] + }`) +} + +// seedIndex writes poolDoc where show reads the cache, and nothing else: +// show takes the document as it stands and does not ask its signature — +// that is verify's question, not the reading form's. +func seedIndex(t *testing.T) string { + t.Helper() + dir := t.TempDir() + poolDir := filepath.Join(dir, "pool") + if err := os.MkdirAll(poolDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(poolDir, "doc.json"), poolDoc(), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +// seedOutbox records two rows the way a run would: through the outbox's own +// open and append, closed again, so status reads what stands on disk. +func seedOutbox(t *testing.T) string { + t.Helper() + dir := t.TempDir() + box, err := outbox.Open(filepath.Join(dir, "pool", "outbox.jsonl")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{`{"n":1}`, `{"n":2}`} { + if err := box.Append([]byte(row)); err != nil { + t.Fatal(err) + } + } + if err := box.Close(); err != nil { + t.Fatal(err) + } + return dir +} + +// The reading form answers over the whole config, every value beside the word +// saying where it came from — and a machine that has never read an index says +// so in a sentence rather than printing nothing at all. An install that has +// recorded nothing of its own says so too. +func TestPoolShowPrintsTheConfigAndSaysWhenNoIndexIsCached(t *testing.T) { + var out strings.Builder + if err := runPoolWith(nil, &out, t.TempDir(), poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + body := out.String() + for _, want := range []string{ + "mode on · default", + "relay https://codeaf.agentfield.ai/pool · default", + "index https://codeaf.agentfield.ai/pool/index.json · default", + "mirror https://raw.githubusercontent.com/Agent-Field/CodeAF/model-pool/pool/index.json · default", + "submit https://codeaf.agentfield.ai/pool/v1/rows · default", + "ttl 1d · default", + "no index cached yet · built-in seed of 2026-09-17", + "own sheet: none", + } { + if !strings.Contains(body, want) { + t.Errorf("show is missing %q:\n%s", want, body) + } + } +} + +// The pin is a source the reader can check, and the word it holds is the mode +// the machine answers to — the same three words the row and CI spell. +func TestPoolShowNamesTheEnvPinAsTheModeSource(t *testing.T) { + var out strings.Builder + if err := runPoolWith([]string{"show"}, &out, t.TempDir(), poolClock(t), + oneEnv("CODEAF_MODEL_POOL", "off")); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "mode off · env") { + t.Fatalf("the env pin did not name itself:\n%s", out.String()) + } +} + +// --json is one parseable object, the cached index under index or null — and, +// this is the point of the absent fields, status's three are absent from a +// show, so a script reads absence as not-asked rather than as zero. +func TestPoolShowJSONWithASeededIndexReportsTheDocument(t *testing.T) { + var out strings.Builder + if err := runPoolWith([]string{"show", "--json"}, &out, seedIndex(t), poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + var answer struct { + Mode string `json:"mode"` + ModeSource string `json:"mode_source"` + IndexURL string `json:"index_url"` + SubmitURL string `json:"submit_url"` + TTLSeconds int `json:"ttl_seconds"` + Pending *int `json:"pending"` + Index *struct { + Generated string `json:"generated"` + AgeSeconds int `json:"age_seconds"` + Schema int `json:"schema"` + Metrics int `json:"metrics"` + Judges int `json:"judges"` + MinInstalls int `json:"min_installs"` + Source string `json:"source"` + } `json:"index"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("--json did not parse: %v\n%s", err, out.String()) + } + if answer.Mode != "on" || answer.ModeSource != "default" { + t.Fatalf("mode = %q from %q", answer.Mode, answer.ModeSource) + } + if answer.IndexURL == "" || answer.SubmitURL == "" || answer.TTLSeconds != 86400 { + t.Fatalf("the config is not whole: %+v", answer) + } + if answer.Pending != nil { + t.Fatal("a show carried status's pending count") + } + held := answer.Index + if held == nil { + t.Fatal("a seeded index was not reported") + } + // 2026-09-18 midnight less 2026-09-10 midnight: eight days, to the second. + if held.Generated != "2026-09-10" || held.AgeSeconds != 8*24*60*60 { + t.Fatalf("generated = %q, age = %d", held.Generated, held.AgeSeconds) + } + if held.Schema != 1 || held.Metrics != 1 || held.Judges != 1 || held.MinInstalls != 1 { + t.Fatalf("the document's counts moved: %+v", held) + } + if held.Source != "cache" { + t.Fatalf("a cached index did not name its source: %q", held.Source) + } +} + +// status is show plus the outbox and the two doors the mode opens, said in +// words a person reads and carried as fields a script reads. +func TestPoolStatusCountsPendingRowsAndNamesItsDoors(t *testing.T) { + dir := seedOutbox(t) + var out strings.Builder + if err := runPoolWith([]string{"status"}, &out, dir, poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "pending 2 · can send yes · can read yes") { + t.Fatalf("status did not count the seeded rows:\n%s", out.String()) + } + if !strings.Contains(out.String(), "own sheet: none") { + t.Fatalf("status did not say the install has recorded nothing of its own:\n%s", out.String()) + } + + out.Reset() + if err := runPoolWith([]string{"status", "--json"}, &out, dir, poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + var answer struct { + Pending int `json:"pending"` + CanSend bool `json:"can_send"` + CanRead bool `json:"can_read"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("status --json did not parse: %v\n%s", err, out.String()) + } + if answer.Pending != 2 || !answer.CanSend || !answer.CanRead { + t.Fatalf("the doors moved: %+v", answer) + } + + // A machine that has recorded no measurement says zero rather than + // inventing a file to count — and the reading form WRITES NOTHING: status + // reads the outbox by count, because opening one would create it. + quiet := t.TempDir() + out.Reset() + if err := runPoolWith([]string{"status"}, &out, quiet, poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "pending 0") { + t.Fatalf("an absent outbox did not read as zero:\n%s", out.String()) + } + if _, err := os.Stat(filepath.Join(quiet, "pool", "outbox.jsonl")); !os.IsNotExist(err) { + t.Fatal("status created the outbox it was only counting") + } +} + +// seedJudgeLast writes the hook's record the way the hook leaves it, so +// status is read against what stands on disk. +func seedJudgeLast(t *testing.T, dir string, last judgeLast) { + t.Helper() + data, err := json.Marshal(last) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "pool"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "pool", "judge-last.json"), data, 0o600); err != nil { + t.Fatal(err) + } +} + +// status says what the last judge did: which model, which seats it scored — +// or, when none answered, how many were asked and the reason the last one +// failed. With no record at all it says none yet, and a file that does not +// parse reads the same way, still on exit 0. +func TestPoolStatusSaysWhatTheLastJudgeDid(t *testing.T) { + dir := t.TempDir() + moment := time.Date(2026, 9, 17, 14, 5, 0, 0, time.Local) + + seedJudgeLast(t, dir, judgeLast{ + At: moment, Task: 7, Judge: "other/judge", + Seats: []string{"crew/worker", "crew/high"}, + Scored: []string{"crew/worker", "crew/high"}, + }) + var out strings.Builder + if err := runPoolWith([]string{"status"}, &out, dir, poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "last judge: 14:05 · other/judge · scored crew/worker, crew/high") { + t.Fatalf("status did not say what the last judge did:\n%s", out.String()) + } + + out.Reset() + seedJudgeLast(t, dir, judgeLast{ + At: moment, Task: 7, + Tried: []string{"other/flaky", "other/steady"}, + Seats: []string{"crew/worker", "crew/high"}, + Reason: "the model answered with a 429", + }) + if err := runPoolWith([]string{"status"}, &out, dir, poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "last judge: 14:05 · failed after 2 candidates (other/flaky, …) · the model answered with a 429") { + t.Fatalf("status did not say how the judging failed:\n%s", out.String()) + } + + // No record, and a record that does not parse: the same sentence, and the + // reading form still exits 0. + for name, seed := range map[string]func(t *testing.T, dir string){ + "absent": func(t *testing.T, dir string) {}, + "malformed": func(t *testing.T, dir string) { + if err := os.MkdirAll(filepath.Join(dir, "pool"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "pool", "judge-last.json"), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + }, + } { + t.Run(name, func(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + quiet := t.TempDir() + seed(t, quiet) + var out strings.Builder + if err := runPoolWith([]string{"status"}, &out, quiet, poolClock(t), deadEnv()); err != nil { + t.Fatalf("a reading form failed over a judge record: %v", err) + } + if !strings.Contains(out.String(), "last judge: none yet") { + t.Fatalf("status did not say none yet:\n%s", out.String()) + } + }) + } +} + +// The JSON form carries the record whole — the moment in RFC 3339, the judge, +// the seats — and null when there is none. +func TestPoolStatusJSONCarriesTheLastJudge(t *testing.T) { + moment := time.Date(2026, 9, 17, 14, 5, 0, 0, time.UTC) + dir := t.TempDir() + seedJudgeLast(t, dir, judgeLast{ + At: moment, Task: 7, Judge: "other/judge", + Seats: []string{"crew/worker"}, + Scored: []string{"crew/worker"}, + }) + var out strings.Builder + if err := runPoolWith([]string{"status", "--json"}, &out, dir, poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + var answer struct { + LastJudge *judgeLast `json:"last_judge"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("status --json did not parse: %v\n%s", err, out.String()) + } + if answer.LastJudge == nil { + t.Fatal("a written record read as none") + } + if answer.LastJudge.Judge != "other/judge" || answer.LastJudge.Task != 7 || + strings.Join(answer.LastJudge.Scored, ", ") != "crew/worker" || answer.LastJudge.Reason != "" { + t.Fatalf("the record moved: %+v", answer.LastJudge) + } + if got, err := time.Parse(time.RFC3339, "2026-09-17T14:05:00Z"); err != nil || !answer.LastJudge.At.Equal(got) { + t.Fatalf("the moment is %v, want the record's own in RFC 3339", answer.LastJudge.At) + } + + // No record: the field is null and not absent, so a script can tell the + // two apart. + out.Reset() + if err := runPoolWith([]string{"status", "--json"}, &out, t.TempDir(), poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), `"last_judge":null`) { + t.Fatalf("a missing record was not said as null:\n%s", out.String()) + } +} + +// An empty profile is the state root's own profile, the way every other file +// under the profile resolves — never a directory called "pool" beside wherever +// the command happened to run. +func TestPoolWithNoProfileDirReadsTheStateRoots(t *testing.T) { + root := t.TempDir() + t.Setenv(home.EnvVar, root) + poolDir := filepath.Join(root, "pool") + if err := os.MkdirAll(poolDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(poolDir, "doc.json"), poolDoc(), 0o644); err != nil { + t.Fatal(err) + } + var out strings.Builder + if err := runPoolWith([]string{"show", "--json"}, &out, "", poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), `"generated":"2026-09-10"`) { + t.Fatalf("an empty profile did not read the state root's pool:\n%s", out.String()) + } +} + +// The build's one key is the index signer's: it decoded at init, it is an +// ed25519 public key's length, and it starts the start-up refresh under a +// mode that reads. +func TestPoolPublicKeysCarryTheIndexSignersKey(t *testing.T) { + if len(poolPublicKeys) != 1 { + t.Fatalf("the build carries %d public key(s), want the index signer's one", len(poolPublicKeys)) + } + if len(poolPublicKeys[0]) != ed25519.PublicKeySize { + t.Fatalf("the built-in key is %d bytes, want an ed25519 public key's %d", len(poolPublicKeys[0]), ed25519.PublicKeySize) + } + + started := 0 + prev := poolRefreshGo + poolRefreshGo = func(scope string, fn func()) { started++ } + t.Cleanup(func() { poolRefreshGo = prev }) + startPoolIndexRefresh(context.Background(), t.TempDir(), poolcfg.Resolve("", "", noEnv), poolPublicKeys) + if started != 1 { + t.Fatalf("the build's key started %d refresh(es) under mode on, want 1", started) + } +} + +// The stored key, when one is set, is the only key the puller is handed: a +// word that does not decode is a key nobody can vouch for, so the answer is +// no keys rather than the build's own — and nothing stored hands back the +// build's. poolTrustedKeysErr says which: a word set but not decodable is a +// reason, not an emptiness. +func TestPoolTrustedKeysFollowTheStoredKey(t *testing.T) { + good := base64.StdEncoding.EncodeToString(make(ed25519.PublicKey, ed25519.PublicKeySize)) + got, err := poolTrustedKeysErr(poolcfg.Resolve("", good, noEnv)) + if err != nil || len(got) != 1 || len(got[0]) != ed25519.PublicKeySize { + t.Fatalf("a stored key gave %d key(s) with err %v, want its one and no error", len(got), err) + } + + for name, bad := range map[string]string{ + "not base64": "not a key at all", + "too short": base64.StdEncoding.EncodeToString([]byte("short")), + } { + keys, keyErr := poolTrustedKeysErr(poolcfg.Resolve("", bad, noEnv)) + if keyErr == nil || len(keys) != 0 { + t.Fatalf("%s: a key that does not decode left %d key(s) in hand and no reason", name, len(keys)) + } + } + + built, err := poolTrustedKeysErr(poolcfg.Resolve("", "", noEnv)) + if err != nil || len(built) != 1 || !bytes.Equal(built[0], poolPublicKeys[0]) { + t.Fatalf("with nothing stored, the build's own key is the key (err %v)", err) + } +} + +// The key ships in the binary, so verify fetches under it: a dead address is +// a fetch that failed — exit 1, in the puller's words — and not a refusal at +// the door, and nothing was cached in its place. +func TestPoolVerifyFetchesUnderTheBuiltInKeyAndNamesWhatFailed(t *testing.T) { + dir := t.TempDir() + // A dead port is the fetch that would have happened: if the door let one + // through, the failure below would be a fetch error, not the sentence. + lookup := oneEnv("CODEAF_MODEL_POOL_URL", "http://127.0.0.1:1/index.json") + var out strings.Builder + err := runPoolWith([]string{"verify"}, &out, dir, poolClock(t), lookup) + if err == nil { + t.Fatal("a dead relay verified") + } + if errors.Is(err, exitIncomplete) { + t.Fatalf("verify refused at the door though the build carries a key: %v", err) + } + if !strings.Contains(err.Error(), "pull") { + t.Fatalf("the failure did not name the fetch: %v", err) + } + if _, statErr := os.Stat(filepath.Join(dir, "pool", "doc.json")); !os.IsNotExist(statErr) { + t.Fatal("the failed verify wrote a cache") + } +} + +// OFF is a refusal at the door even with a key in hand, because the setting +// is the word the machine answers to. +func TestPoolVerifyRefusesWhenThePoolIsOff(t *testing.T) { + var out strings.Builder + err := runPoolWith([]string{"verify"}, &out, t.TempDir(), poolClock(t), + oneEnv("CODEAF_MODEL_POOL", "off")) + if !errors.Is(err, exitIncomplete) { + t.Fatalf("an off pool did not refuse on exit 2: %v", err) + } + if !strings.Contains(out.String(), "the Model Pool is off in settings") { + t.Fatalf("the refusal did not say so:\n%s", out.String()) + } +} + +// A key the flag cannot read — not base64, or not the length of an ed25519 +// public key — is refused where it was typed, by the flag set, on exit 1, +// because nothing was attempted. +func TestPoolVerifyRefusesAKeyItCannotRead(t *testing.T) { + for _, bad := range []string{"not base64!!", "abcd"} { + var out strings.Builder + err := runPoolWith([]string{"verify", "--key", bad}, &out, t.TempDir(), poolClock(t), noEnv) + if !errors.Is(err, exitCannotRun) { + t.Fatalf("a key %q is a door refusal, not a fetch: %v", bad, err) + } + } +} + +// --key replaces the keys the build resolves, it does not add to them: the +// document below is signed under the stored key, which is the key the verb +// would trust with no flag at all — the stand-in for the build's own, whose +// private half no test holds. A fresh unrelated key must be able to prove a +// document does not verify under it, which is the whole reason the flag +// exists; were the flag's keys only added to the resolved ones, the stored +// key would still be in the list and the document would verify. The control +// run without the flag verifies, so the refusal is the flag's doing and not +// the signature's. +func TestPoolVerifyChecksOnlyUnderTheKeyItIsGiven(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + server := poolServer(t, priv, signedPoolDoc(7)) + other, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", + "CODEAF_MODEL_POOL_MIRROR_URL": "", + }) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.json"), + []byte(`{"models.pool.public_key": "`+base64.StdEncoding.EncodeToString(pub)+`"}`), 0o644); err != nil { + t.Fatal(err) + } + var out strings.Builder + err = runPoolWith([]string{"verify", "--key", base64.StdEncoding.EncodeToString(other)}, + &out, dir, poolClock(t), lookup) + if err == nil || !strings.Contains(err.Error(), "pull: signature does not verify") { + t.Fatalf("a document the --key does not trust verified: %v", err) + } + if _, statErr := os.Stat(filepath.Join(dir, "pool", "doc.json")); !os.IsNotExist(statErr) { + t.Fatal("a refused document was cached") + } + + // The control: no flag, the stored key answers for the same document. + out.Reset() + if err := runPoolWith([]string{"verify"}, &out, dir, poolClock(t), lookup); err != nil { + t.Fatalf("the stored key did not verify its own document: %v", err) + } + if !strings.Contains(out.String(), "signature good") { + t.Fatalf("the control run did not verify:\n%s", out.String()) + } +} + +// A stored key that does not decode is not the ordinary nothing: verify +// refuses at the door with the row's name on it, and nothing is fetched. +func TestPoolVerifyNamesAStoredKeyThatDoesNotDecode(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.json"), + []byte(`{"models.pool.public_key": "not-a-key"}`), 0o644); err != nil { + t.Fatal(err) + } + var out strings.Builder + err := runPoolWith([]string{"verify"}, &out, dir, poolClock(t), deadEnv()) + if !errors.Is(err, exitIncomplete) { + t.Fatalf("a stored key that does not decode is a door refusal, not a fetch: %v", err) + } + if !strings.Contains(out.String(), "models.pool.public_key does not decode") { + t.Fatalf("the refusal did not name the row:\n%s", out.String()) + } +} + +// The same refusal names the environment pin when the broken word came in as +// one — the remedy belongs to whichever word is in force. +func TestPoolVerifyNamesTheEnvPinWhenItsKeyDoesNotDecode(t *testing.T) { + var out strings.Builder + err := runPoolWith([]string{"verify"}, &out, t.TempDir(), poolClock(t), + oneEnv("CODEAF_MODEL_POOL_PUBLIC_KEY", "not-a-key")) + if !errors.Is(err, exitIncomplete) { + t.Fatalf("a pin that does not decode is a door refusal, not a fetch: %v", err) + } + if !strings.Contains(out.String(), "CODEAF_MODEL_POOL_PUBLIC_KEY does not decode") { + t.Fatalf("the refusal did not name the pin:\n%s", out.String()) + } +} + +// The happy path, over no network at all: an index and its signature on disk, +// fetched through the same Puller a real verify uses, checked under the key +// --key hands in, and read back as the sentence a person came for. The fetch +// is a pull, so what verified is the cache show reads next. +func TestPoolVerifyFetchesAndChecksASignedIndex(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "index.json"), poolDoc(), 0o644); err != nil { + t.Fatal(err) + } + sig := ed25519.Sign(priv, poolDoc()) + if err := os.WriteFile(filepath.Join(src, "index.json.sig"), + []byte(base64.StdEncoding.EncodeToString(sig)), 0o644); err != nil { + t.Fatal(err) + } + lookup := oneEnv("CODEAF_MODEL_POOL_URL", filepath.Join(src, "index.json")) + dir := t.TempDir() + var out strings.Builder + if err := runPoolWith([]string{"verify", "--key", base64.StdEncoding.EncodeToString(pub)}, + &out, dir, poolClock(t), lookup); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "signature good: version 7, generated 2026-09-10, 1 metric") { + t.Fatalf("verify did not read the fetched document:\n%s", out.String()) + } + out.Reset() + if err := runPoolWith([]string{"show"}, &out, dir, poolClock(t), lookup); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "generated 2026-09-10") { + t.Fatalf("a verified index did not land in the cache show reads:\n%s", out.String()) + } +} + +// A document the key does not trust is not verified: the failure says so in +// the puller's words on exit 1, and — this is the puller's own promise, read +// through the door — nothing wrong is cached in its place. +func TestPoolVerifyRefusesADocumentItsKeyDoesNotTrust(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + src := t.TempDir() + doc := append([]byte(nil), poolDoc()...) + // One bit of the generated date, so the bytes no longer match the + // signature beside them and nothing else about the document moved. + doc[46] ^= 0x01 + if err := os.WriteFile(filepath.Join(src, "index.json"), doc, 0o644); err != nil { + t.Fatal(err) + } + sig := ed25519.Sign(priv, poolDoc()) + if err := os.WriteFile(filepath.Join(src, "index.json.sig"), + []byte(base64.StdEncoding.EncodeToString(sig)), 0o644); err != nil { + t.Fatal(err) + } + lookup := oneEnv("CODEAF_MODEL_POOL_URL", filepath.Join(src, "index.json")) + dir := t.TempDir() + var out strings.Builder + err = runPoolWith([]string{"verify", "--key", base64.StdEncoding.EncodeToString(pub)}, + &out, dir, poolClock(t), lookup) + if err == nil || !strings.Contains(err.Error(), "pull: signature does not verify") { + t.Fatalf("a tampered document did not refuse: %v", err) + } + if _, statErr := os.Stat(filepath.Join(dir, "pool", "doc.json")); !os.IsNotExist(statErr) { + t.Fatal("a refused document was cached") + } +} + +// ── THE SEATED INDEX ──────────────────────────────────────────────────────── + +// writePoolDoc puts a document where poolIndexFor reads the cache: doc.json +// under the profile's pool directory. +func writePoolDoc(t *testing.T, dir, doc string) { + t.Helper() + poolDir := filepath.Join(dir, "pool") + if err := os.MkdirAll(poolDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(poolDir, "doc.json"), []byte(doc), 0o644); err != nil { + t.Fatal(err) + } +} + +// With no cache the reader answers the seed this build carries, and a worker +// cell is in it — so `learn` has numbers on day one. +func TestPoolIndexForAnswersTheSeedWithNoCache(t *testing.T) { + held := poolIndexFor(t.TempDir(), poolcfg.Resolve("", "", noEnv), poolClock(t))() + if held == nil { + t.Fatal("no cache and no seed: the reader answered nothing") + } + worker := false + for _, cell := range held.Cells("role_quality") { + if cell.Role == "worker" { + worker = true + } + } + if !worker { + t.Fatal("the seed answered no worker cell") + } +} + +// A cached document whose generated day is newer than the seed's wins: the +// reader hands back the cached numbers rather than the embedded ones. +func TestPoolIndexForKeepsANewerCache(t *testing.T) { + dir := t.TempDir() + writePoolDoc(t, dir, `{ + "schema": 1, + "generated": "2026-09-20", + "min_installs": 1, + "metrics": {"role_quality": {"kind": "gaussian", "dims": ["role", "model"]}}, + "cells": [{"metric": "role_quality", "role": "worker", "model": "z-ai/glm-5.3", "mean": 75, "sd": 7, "n": 30}] + }`) + held := poolIndexFor(dir, poolcfg.Resolve("", "", noEnv), poolClock(t))() + if held == nil || held.Generated().Format("2006-01-02") != "2026-09-20" { + t.Fatalf("a newer cache did not win: %v", held) + } +} + +// A cache that does not parse is not a cache: the seed stands in its place. +func TestPoolIndexForIgnoresAnUnparsableCache(t *testing.T) { + dir := t.TempDir() + writePoolDoc(t, dir, "{ this is not a document") + held := poolIndexFor(dir, poolcfg.Resolve("", "", noEnv), poolClock(t))() + if held == nil || held.Generated().Format("2006-01-02") != "2026-09-17" { + t.Fatalf("an unparsable cache did not fall back to the seed: %v", held) + } +} + +// A mode that forbids reading answers no index at all. +func TestPoolIndexForAnswersNothingWhenTheModeIsOff(t *testing.T) { + if held := poolIndexFor(t.TempDir(), poolcfg.Resolve("off", "", noEnv), poolClock(t))(); held != nil { + t.Fatal("a mode that forbids reading answered an index") + } +} + +// No key in the build means no fetch: the refresh starts no goroutine, which +// is every run on a build with no key compiled in. A key starts the one fetch, +// and the mode still has to allow reading. +func TestPoolRefreshStartsNoGoroutineWithoutAKey(t *testing.T) { + started := 0 + prev := poolRefreshGo + poolRefreshGo = func(scope string, fn func()) { started++ } + t.Cleanup(func() { poolRefreshGo = prev }) + + key := []ed25519.PublicKey{make(ed25519.PublicKey, ed25519.PublicKeySize)} + on := poolcfg.Resolve("", "", noEnv) + off := poolcfg.Resolve("off", "", noEnv) + + startPoolIndexRefresh(context.Background(), t.TempDir(), on, nil) + if started != 0 { + t.Fatalf("no key, yet %d goroutine(s) started", started) + } + startPoolIndexRefresh(context.Background(), t.TempDir(), off, key) + if started != 0 { + t.Fatal("a mode that forbids reading started a goroutine") + } + startPoolIndexRefresh(context.Background(), t.TempDir(), on, key) + if started != 1 { + t.Fatalf("with a key, %d goroutine(s) started, want 1", started) + } +} + +// stubPoolRefresh stands the goroutine guard in for the length of one test +// and answers the counter it increments, so a test of what wirePoolIndex +// seats runs no fetch and no push. The tests that call it read seats, not +// errands; the errands have their own tests. +func stubPoolRefresh(t *testing.T) *int { + t.Helper() + started := 0 + prev := poolRefreshGo + poolRefreshGo = func(scope string, fn func()) { started++ } + t.Cleanup(func() { poolRefreshGo = prev }) + return &started +} + +// The start-up errands are one push and one refresh behind the same guard: +// a mode that sends and reads starts both, a mode that only reads starts +// only the refresh (its push would send nowhere), and off starts neither. +func TestWirePoolIndexStartsTheRefreshAndThePush(t *testing.T) { + started := stubPoolRefresh(t) + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/v1/rows") + + // The default mode is on: the refresh and the push both start. + wirePoolIndex(t.TempDir()) + if *started != 2 { + t.Fatalf("a sending pool started %d errand(s) at start-up, want the refresh and the push", *started) + } + + // A mode that does not send starts the refresh alone. + *started = 0 + t.Setenv("CODEAF_MODEL_POOL", "read") + wirePoolIndex(t.TempDir()) + if *started != 1 { + t.Fatalf("a read-only pool started %d errand(s), want the refresh alone", *started) + } +} + +// With no cache, the --json shape reports the seed the build carries and says +// so under source, so a script sees the index a pick would read. +func TestPoolShowJSONWithNoCacheReportsTheSeed(t *testing.T) { + var out strings.Builder + if err := runPoolWith([]string{"show", "--json"}, &out, t.TempDir(), poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + var answer struct { + Index *struct { + Generated string `json:"generated"` + Source string `json:"source"` + } `json:"index"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("--json did not parse: %v\n%s", err, out.String()) + } + if answer.Index == nil || answer.Index.Source != "seed" || answer.Index.Generated != "2026-09-17" { + t.Fatalf("the seed was not reported: %+v", answer.Index) + } +} + +// The reading form counts the held document's cells, so a person can tell +// an empty document from a full one without opening it. The two cells name +// no model; the point is the count, on the index line and in --json alike. +func TestPoolShowSaysHowManyCellsTheCachedIndexHolds(t *testing.T) { + dir := t.TempDir() + poolDir := filepath.Join(dir, "pool") + if err := os.MkdirAll(poolDir, 0o755); err != nil { + t.Fatal(err) + } + doc := []byte(`{ + "version": 7, + "schema": 1, + "generated": "2026-09-10", + "min_installs": 1, + "judges": ["z-ai/glm-5.3"], + "metrics": {"role_rating": {"kind": "gaussian", "dims": ["role", "model"]}}, + "cells": [ + {"metric": "role_rating", "role": "planner", "model": "z-ai/glm-5.3", "mean": 1312, "sd": 18, "n": 9}, + {"metric": "role_rating", "role": "checker", "model": "z-ai/glm-5.3", "mean": 1290, "sd": 21, "n": 9} + ] + }`) + if err := os.WriteFile(filepath.Join(poolDir, "doc.json"), doc, 0o644); err != nil { + t.Fatal(err) + } + + var out strings.Builder + if err := runPoolWith([]string{"show"}, &out, dir, poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + body := out.String() + indexLine := "" + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, "index ·") { + indexLine = line + } + } + if indexLine == "" { + t.Fatalf("show printed no index line:\n%s", body) + } + if !strings.Contains(indexLine, "2 cells") { + t.Errorf("the index line did not count its cells:\n%s", indexLine) + } + + out.Reset() + if err := runPoolWith([]string{"show", "--json"}, &out, dir, poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + var answer struct { + Index *struct { + Cells int `json:"cells"` + } `json:"index"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("--json did not parse: %v\n%s", err, out.String()) + } + if answer.Index == nil || answer.Index.Cells != 2 { + t.Fatalf("--json carried %+v, want 2 cells", answer.Index) + } +} + +// With no cache the reading form names the seed's own cell count, computed +// here from the seed itself rather than hard-coded, so the sentence follows +// the document the build carries. +func TestPoolShowSaysHowManyCellsTheBuiltInSeedHolds(t *testing.T) { + seed, err := index.SeedIndex() + if err != nil { + t.Fatal(err) + } + total := 0 + for _, metric := range seed.Metrics() { + total += len(seed.Cells(metric)) + } + + var out strings.Builder + if err := runPoolWith(nil, &out, t.TempDir(), poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + if want := countWord(total, "cell", "cells"); !strings.Contains(out.String(), want) { + t.Errorf("show did not say the seed holds %q:\n%s", want, out.String()) + } +} + +// ── THE OWN SHEET ─────────────────────────────────────────────────────────── + +// seedOwnSheet records the install's own scores the way a recorder would: +// through the sheet's own observe, saved to the path the pool reads it back +// from. +func seedOwnSheet(t *testing.T, dir string) { + t.Helper() + sheet := tally.New() + sheet.Observe("role_quality", "worker", "a/one", nil, 80) + sheet.Observe("role_quality", "worker", "a/one", nil, 90) + sheet.Observe("role_quality", "worker", "b/two", nil, 70) + if err := record.SaveSheet(filepath.Join(dir, "pool", "own.json"), sheet); err != nil { + t.Fatal(err) + } +} + +// The install's own judged scores are said with their noun — the cells the +// picker reads beside the index and the observations behind them — on the +// reading forms and in the --json object alike. +func TestPoolShowSaysWhatTheOwnSheetHolds(t *testing.T) { + dir := t.TempDir() + seedOwnSheet(t, dir) + + var out strings.Builder + if err := runPoolWith([]string{"show"}, &out, dir, poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "own sheet: 2 cells, 3 observations") { + t.Fatalf("show did not count the own sheet's cells:\n%s", out.String()) + } + + out.Reset() + if err := runPoolWith([]string{"status"}, &out, dir, poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "own sheet: 2 cells, 3 observations") { + t.Fatalf("status did not count the own sheet's cells:\n%s", out.String()) + } + + out.Reset() + if err := runPoolWith([]string{"show", "--json"}, &out, dir, poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + var answer struct { + Own struct { + Cells int `json:"cells"` + Observations int `json:"observations"` + } `json:"own"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("--json did not parse: %v\n%s", err, out.String()) + } + if answer.Own.Cells != 2 || answer.Own.Observations != 3 { + t.Fatalf("the own sheet counted as %+v, want 2 cells over 3 observations", answer.Own) + } +} + +// An own sheet that does not parse is a loss, not a fault a reading form +// stops for: it reads as none. +func TestPoolShowReadsAnUnparsableOwnSheetAsNone(t *testing.T) { + dir := t.TempDir() + poolDir := filepath.Join(dir, "pool") + if err := os.MkdirAll(poolDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(poolDir, "own.json"), []byte("not a document"), 0o600); err != nil { + t.Fatal(err) + } + var out strings.Builder + if err := runPoolWith([]string{"show"}, &out, dir, poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "own sheet: none") { + t.Fatalf("a broken own sheet did not read as none:\n%s", out.String()) + } +} + +// The wired reader answers the own sheet's cells — the install's own evidence, +// read once and parsed once — and an off mode seats nothing at all. +func TestWirePoolIndexSeatsTheOwnSheetsCells(t *testing.T) { + prevIndex, prevOwn := config.AutoIndex, config.AutoOwnCells + t.Cleanup(func() { config.AutoIndex, config.AutoOwnCells = prevIndex, prevOwn }) + stubPoolRefresh(t) + + dir := t.TempDir() + seedOwnSheet(t, dir) + wirePoolIndex(dir) + if config.AutoOwnCells == nil { + t.Fatal("the own sheet was not seated") + } + own := config.AutoOwnCells() + want := []crewpick.Cell{ + {Role: "worker", Model: "a/one", Mean: 85, N: 2}, + {Role: "worker", Model: "b/two", Mean: 70, N: 1}, + } + if len(own) != len(want) { + t.Fatalf("the seated cells are %+v, want %+v", own, want) + } + for i := range want { + if own[i] != want[i] { + t.Fatalf("cell %d is %+v, want %+v", i, own[i], want[i]) + } + } + if config.AutoIndex == nil || config.AutoIndex() == nil { + t.Fatal("the index was not seated beside the own sheet") + } +} + +// A mode that forbids reading seats nothing: the own sheet is the pool's own +// reading, and off is off for the whole of it. +func TestWirePoolIndexSeatsNothingWhenThePoolIsOff(t *testing.T) { + prevIndex, prevOwn := config.AutoIndex, config.AutoOwnCells + t.Cleanup(func() { config.AutoIndex, config.AutoOwnCells = prevIndex, prevOwn }) + t.Setenv("CODEAF_MODEL_POOL", "off") + stubPoolRefresh(t) + + dir := t.TempDir() + seedOwnSheet(t, dir) + wirePoolIndex(dir) + if config.AutoOwnCells != nil { + t.Fatal("a pool that forbids reading seated the own sheet") + } + if config.AutoIndex != nil && config.AutoIndex() != nil { + t.Fatal("a pool that forbids reading seated an index") + } +} + +// An own sheet that does not parse is a loss, not a fault a pick stops for: +// the wired reader answers nothing rather than a broken sheet's half. +func TestWirePoolIndexSeatsNothingForAnUnparsableOwnSheet(t *testing.T) { + prevOwn := config.AutoOwnCells + t.Cleanup(func() { config.AutoOwnCells = prevOwn }) + stubPoolRefresh(t) + + dir := t.TempDir() + poolDir := filepath.Join(dir, "pool") + if err := os.MkdirAll(poolDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(poolDir, "own.json"), []byte("not a document"), 0o600); err != nil { + t.Fatal(err) + } + wirePoolIndex(dir) + if got := config.AutoOwnCells; got != nil && got() != nil { + t.Fatal("a broken own sheet was seated") + } +} + +// ── THE MIRROR ────────────────────────────────────────────────────────────── + +// poolEnv is an environment holding exactly the names given, so a test pins +// the pool's addresses without touching the process. +func poolEnv(pairs map[string]string) func(string) (string, bool) { + return func(name string) (string, bool) { + value, set := pairs[name] + return value, set + } +} + +// deadEnv pins both index addresses at a closed local port, so a probe fails +// at once and reaches no network. +func deadEnv() func(string) (string, bool) { + return poolEnv(map[string]string{ + "CODEAF_MODEL_POOL_URL": "http://127.0.0.1:1/index.json", + "CODEAF_MODEL_POOL_MIRROR_URL": "http://127.0.0.1:1/index.json", + }) +} + +// signedPoolDoc is a document the puller accepts: one integer version and +// nothing else it reads. The version is where a test finds which address was +// the one that answered. +func signedPoolDoc(version int) []byte { + return []byte(fmt.Sprintf(`{"version": %d, "schema": 1, "generated": "2026-09-10", "min_installs": 1, "judges": [], "metrics": {}, "cells": []}`, version)) +} + +// poolServer serves a signed index over http the way a relay does: the +// document at /index.json and its base64 signature beside it at +// /index.json.sig, both signed under priv. +func poolServer(t *testing.T, priv ed25519.PrivateKey, doc []byte) *httptest.Server { + t.Helper() + sig := base64.StdEncoding.EncodeToString(ed25519.Sign(priv, doc)) + mux := http.NewServeMux() + mux.HandleFunc("/index.json", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(doc) + }) + mux.HandleFunc("/index.json.sig", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(sig)) + }) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + return server +} + +// cachedVersion reads the "version" of the document under the profile's pool +// directory, and whether one is there at all. +func cachedVersion(t *testing.T, dir string) (int64, bool) { + t.Helper() + doc, err := os.ReadFile(filepath.Join(dir, "pool", "doc.json")) + if err != nil { + return 0, false + } + var obj struct { + Version int64 `json:"version"` + } + if err := json.Unmarshal(doc, &obj); err != nil { + t.Fatalf("the cached document does not parse: %v", err) + } + return obj.Version, true +} + +// A relay that does not answer is not the end of the fetch: the mirror is +// asked with the same keys and the same cache directory, and the document it +// serves lands where the next start reads it. +func TestPoolRefreshFallsToTheMirrorWhenTheRelayIsDown(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + mirror := poolServer(t, priv, signedPoolDoc(9)) + dir := t.TempDir() + cfg := poolcfg.Config{Mode: poolcfg.On, IndexURL: "http://127.0.0.1:1/index.json", MirrorURL: mirror.URL + "/index.json"} + refreshPoolIndex(context.Background(), dir, cfg, []ed25519.PublicKey{pub}) + version, ok := cachedVersion(t, dir) + if !ok || version != 9 { + t.Fatalf("the mirror's document was not kept: version %d, cached %v", version, ok) + } +} + +// Both addresses down leaves the cache exactly where it was: the fetch falls +// back to the copy already on disk and writes nothing. +func TestPoolRefreshKeepsTheCacheWhenBothAddressesAreDown(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + relay := poolServer(t, priv, signedPoolDoc(7)) + dir := t.TempDir() + live := poolcfg.Config{Mode: poolcfg.On, IndexURL: relay.URL + "/index.json", MirrorURL: relay.URL + "/index.json"} + refreshPoolIndex(context.Background(), dir, live, []ed25519.PublicKey{pub}) + if version, ok := cachedVersion(t, dir); !ok || version != 7 { + t.Fatalf("the relay's document was not cached: version %d, cached %v", version, ok) + } + relay.Close() + dead := poolcfg.Config{Mode: poolcfg.On, IndexURL: "http://127.0.0.1:1/index.json", MirrorURL: "http://127.0.0.1:1/index.json"} + refreshPoolIndex(context.Background(), dir, dead, []ed25519.PublicKey{pub}) + if version, ok := cachedVersion(t, dir); !ok || version != 7 { + t.Fatalf("a failed refresh moved the cache: version %d, cached %v", version, ok) + } +} + +// A signature failure is a statement about the primary's bytes, not a dead +// source: the mirror is not asked, so its good document is never cached in +// place of the one whose signature just failed. +func TestPoolRefreshDoesNotFallToTheMirrorOnABadSignature(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + _, otherPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + // The primary serves a document signed under a key nobody trusts; the + // mirror serves a good one that must not be reached. + relay := poolServer(t, otherPriv, signedPoolDoc(7)) + mirror := poolServer(t, priv, signedPoolDoc(9)) + dir := t.TempDir() + cfg := poolcfg.Config{Mode: poolcfg.On, IndexURL: relay.URL + "/index.json", MirrorURL: mirror.URL + "/index.json"} + refreshPoolIndex(context.Background(), dir, cfg, []ed25519.PublicKey{pub}) + if _, ok := cachedVersion(t, dir); ok { + t.Fatal("a document with a bad signature fell through to the mirror, or was cached") + } +} + +// ── THE RELAY LINE ────────────────────────────────────────────────────────── + +// status asks the relay whether it answers, over a signed document served by +// an httptest relay and checked under the key --key hands in. +func TestPoolStatusSaysTheRelayAnswered(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + server := poolServer(t, priv, signedPoolDoc(7)) + lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", + "CODEAF_MODEL_POOL_MIRROR_URL": "", + }) + var out strings.Builder + if err := runPoolWith([]string{"status", "--key", base64.StdEncoding.EncodeToString(pub)}, &out, t.TempDir(), poolClock(t), lookup); err != nil { + t.Fatalf("a reachable relay failed status: %v", err) + } + if !strings.Contains(out.String(), "relay: reachable · index version 7") { + t.Fatalf("status did not say the relay answered:\n%s", out.String()) + } +} + +// A closed relay and a closed mirror are a reading, not a failure: status says +// both are unreachable, says what it read instead, and still exits 0. +func TestPoolStatusSaysUnreachableAndStillExitsZero(t *testing.T) { + var out strings.Builder + if err := runPoolWith([]string{"status"}, &out, t.TempDir(), poolClock(t), deadEnv()); err != nil { + t.Fatalf("an unreachable relay failed status: %v", err) + } + body := out.String() + if !strings.Contains(body, "relay: unreachable (") { + t.Fatalf("status did not say the relay was unreachable:\n%s", body) + } + if !strings.Contains(body, "mirror: unreachable (") { + t.Fatalf("status did not say the mirror was unreachable:\n%s", body) + } + if !strings.Contains(body, "reading built-in seed") { + t.Fatalf("status did not say what it read instead:\n%s", body) + } +} + +// The mirror is asked when the relay does not answer, and the line says so. +func TestPoolStatusFallsToTheMirrorWhenTheRelayIsDown(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + mirror := poolServer(t, priv, signedPoolDoc(7)) + lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL_URL": "http://127.0.0.1:1/index.json", + "CODEAF_MODEL_POOL_MIRROR_URL": mirror.URL + "/index.json", + }) + var out strings.Builder + if err := runPoolWith([]string{"status", "--key", base64.StdEncoding.EncodeToString(pub)}, &out, t.TempDir(), poolClock(t), lookup); err != nil { + t.Fatalf("a mirror that answered still failed status: %v", err) + } + body := out.String() + if !strings.Contains(body, "relay: unreachable (") || !strings.Contains(body, "mirror: reachable · index version 7") { + t.Fatalf("status did not fall to the mirror:\n%s", body) + } +} + +// status checks under --key alone too: the relay's document is signed under +// the stored key — the one the probe would trust with no flag at all — and a +// fresh unrelated key must be able to prove it does not answer. Were the +// flag's keys only added to the resolved ones, the stored key would still be +// in the list and the relay would answer; the control run without the flag +// does answer, so the refusal is the flag's doing. +func TestPoolStatusChecksOnlyUnderTheKeyItIsGiven(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + server := poolServer(t, priv, signedPoolDoc(7)) + other, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", + "CODEAF_MODEL_POOL_MIRROR_URL": "", + }) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.json"), + []byte(`{"models.pool.public_key": "`+base64.StdEncoding.EncodeToString(pub)+`"}`), 0o644); err != nil { + t.Fatal(err) + } + var out strings.Builder + if err := runPoolWith([]string{"status", "--json", "--key", base64.StdEncoding.EncodeToString(other)}, &out, dir, poolClock(t), lookup); err != nil { + t.Fatalf("a reading form failed over a signature that did not check: %v", err) + } + var answer struct { + Relay *probeSummary `json:"relay"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("status --json did not parse: %v\n%s", err, out.String()) + } + if answer.Relay == nil || answer.Relay.Reachable || !strings.Contains(answer.Relay.Reason, "signature does not verify") { + t.Fatalf("the relay was not checked under the flag's key alone: %+v", answer.Relay) + } + + // The control: no flag, the same relay answers under the stored key. + out.Reset() + if err := runPoolWith([]string{"status", "--json"}, &out, dir, poolClock(t), lookup); err != nil { + t.Fatalf("the stored key did not answer for its own document: %v", err) + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("status --json did not parse: %v\n%s", err, out.String()) + } + if answer.Relay == nil || !answer.Relay.Reachable { + t.Fatalf("the control run did not reach the relay under the stored key: %+v", answer.Relay) + } +} + +// A stored key that does not decode does not fail a reading: status says why +// the relay was not asked, in the line and in the JSON reason, and exits 0. +func TestPoolStatusSaysWhyAStoredKeyDoesNotDecode(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.json"), + []byte(`{"models.pool.public_key": "not-a-key"}`), 0o644); err != nil { + t.Fatal(err) + } + var out strings.Builder + if err := runPoolWith([]string{"status"}, &out, dir, poolClock(t), deadEnv()); err != nil { + t.Fatalf("a broken stored key failed status: %v", err) + } + if !strings.Contains(out.String(), "relay: unreachable (models.pool.public_key does not decode") { + t.Fatalf("status did not name the broken row:\n%s", out.String()) + } + + out.Reset() + if err := runPoolWith([]string{"status", "--json"}, &out, dir, poolClock(t), deadEnv()); err != nil { + t.Fatal(err) + } + var answer struct { + Relay *probeSummary `json:"relay"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("status --json did not parse: %v\n%s", err, out.String()) + } + if answer.Relay == nil || !strings.Contains(answer.Relay.Reason, "models.pool.public_key does not decode") { + t.Fatalf("the reason did not name the broken row: %+v", answer.Relay) + } +} + +// A mode that forbids reading asks the network nothing and says so, whatever +// the addresses in force are. +func TestPoolStatusDoesNotReadWhenOff(t *testing.T) { + var out strings.Builder + lookup := poolEnv(map[string]string{"CODEAF_MODEL_POOL": "off"}) + if err := runPoolWith([]string{"status"}, &out, t.TempDir(), poolClock(t), lookup); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "relay: not read (model_pool off)") { + t.Fatalf("an off pool did not say it asked nothing:\n%s", out.String()) + } +} + +// status --json carries the relay and the mirror as objects — whether each +// answered, the version it served, and the reason when it did not. +func TestPoolStatusJSONCarriesTheRelayAndMirror(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + server := poolServer(t, priv, signedPoolDoc(7)) + lookup := poolEnv(map[string]string{ + "CODEAF_MODEL_POOL_URL": server.URL + "/index.json", + "CODEAF_MODEL_POOL_MIRROR_URL": "", + }) + var out strings.Builder + if err := runPoolWith([]string{"status", "--json", "--key", base64.StdEncoding.EncodeToString(pub)}, &out, t.TempDir(), poolClock(t), lookup); err != nil { + t.Fatal(err) + } + var answer struct { + Relay *probeSummary `json:"relay"` + Mirror *probeSummary `json:"mirror"` + } + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("status --json did not parse: %v\n%s", err, out.String()) + } + if answer.Relay == nil || !answer.Relay.Reachable || answer.Relay.Version != 7 { + t.Fatalf("the relay was not reported: %+v", answer.Relay) + } + if answer.Mirror == nil || answer.Mirror.Reachable { + t.Fatalf("the mirror was reported as answering though the relay did: %+v", answer.Mirror) + } +} + +// A show asks nothing, so its object carries neither a relay nor a mirror. +func TestPoolShowJSONCarriesNoRelayAnswer(t *testing.T) { + var out strings.Builder + if err := runPoolWith([]string{"show", "--json"}, &out, t.TempDir(), poolClock(t), noEnv); err != nil { + t.Fatal(err) + } + var answer map[string]json.RawMessage + if err := json.Unmarshal([]byte(out.String()), &answer); err != nil { + t.Fatalf("show --json did not parse: %v\n%s", err, out.String()) + } + if _, has := answer["relay"]; has { + t.Fatal("a show carried a relay field") + } + if _, has := answer["mirror"]; has { + t.Fatal("a show carried a mirror field") + } +} diff --git a/cmd/codeaf/poolindex.go b/cmd/codeaf/poolindex.go new file mode 100644 index 0000000000..15f433cd51 --- /dev/null +++ b/cmd/codeaf/poolindex.go @@ -0,0 +1,178 @@ +// The Model Pool's index, seated at start-up. +// +// The index is a measurement document the crew picker reads a seat's quality +// from (config.PoolQualityMetric, through config.AutoIndex). A machine that +// has fetched one keeps it under the profile's pool directory; a machine that +// has never fetched one still has the index the binary carries +// (internal/pool/index's Seed), so a tier row that says `auto` has numbers on +// its first run rather than none. +// +// READ ONCE, NEVER ON THE RUN'S PATH. [poolIndexFor] parses the cache beside +// the seed exactly once and hands back a function answering the value it +// already has, so a pick asks no disk and no lock. The one fetch this file +// makes runs in a goroutine started where the loader is seated, and it never +// answers a pick: it writes the puller's own cache and the change is read at +// the NEXT start. A run's picks must not move under it. +package main + +import ( + "context" + "crypto/ed25519" + "errors" + "log" + "os" + "path/filepath" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/crewpick" + "github.com/Agent-Field/codeaf/internal/guard" + "github.com/Agent-Field/codeaf/internal/pool/index" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" + "github.com/Agent-Field/codeaf/internal/pool/pull" + "github.com/Agent-Field/codeaf/internal/pool/record" + "github.com/Agent-Field/codeaf/internal/trace" +) + +// poolIndexFor builds this process's one index reader. The read is +// start-up work and nothing on a run's path: the document is parsed here, +// once, and the returned function answers the value already in hand, so a +// pick over many tiers does no disk and no decode per call. +// +// THE CACHE WINS ONLY WHEN IT IS FRESHER. A cached doc.json is handed to +// index.Fallback beside the embedded seed, which trusts whichever declares +// the newer generated day — a young cache keeps its numbers, an old or +// unparsable one leaves the seed in place. A mode that forbids reading +// (poolcfg.Off) answers no index at all, which is the same nothing the seam +// read before this file existed. +// +// THE ONE ERROR IT CANNOT RECOVER FROM is an embedded seed that does not +// parse; that answers nil rather than panicking, though a test in the index +// package parses the very bytes this build carries. +func poolIndexFor(profileDir string, cfg poolcfg.Config, now func() time.Time) func() *index.Index { + if !cfg.CanRead() { + return func() *index.Index { return nil } + } + var cached []byte + if doc, err := os.ReadFile(filepath.Join(config.ProfilePath(profileDir, "pool"), "doc.json")); err == nil { + cached = doc + } + held, err := index.Fallback(cached, index.Seed()) + if err != nil { + seed, seedErr := index.SeedIndex() + if seedErr != nil { + return func() *index.Index { return nil } + } + held = seed + } + return func() *index.Index { return held } +} + +// poolRefreshGo is [guard.Go] behind a variable so a test can prove that no +// goroutine is started when the build carries no key. +var poolRefreshGo = func(scope string, fn func()) { guard.Go(scope, fn) } + +// startPoolIndexRefresh is the loader's own tail: it starts the background +// fetch that keeps the cache fresh, and starts NOTHING when the mode forbids +// reading or the build carries no public key to check a fetched document under +// — which, on a build with no key compiled in, is every run. +func startPoolIndexRefresh(ctx context.Context, profileDir string, cfg poolcfg.Config, keys []ed25519.PublicKey) { + if !cfg.CanRead() || len(keys) == 0 { + return + } + poolRefreshGo("pool/index", func() { refreshPoolIndex(ctx, profileDir, cfg, keys) }) +} + +// refreshPoolIndex fetches a fresh index and lets the puller's own cache keep +// it for the NEXT start. A changed document is NOT swapped into the running +// process: a run's picks must not move under it, so a pick here always reads +// the index this process was seated with, and the fresh one is read at the +// next start. +// +// It is quiet by design. A dead host, a timeout or a signature that does not +// check is an ordinary state for a fetch nothing waited for, so the error is +// said only under the debug record's switch and never on the surface. +// +// THE MIRROR IS ASKED WHEN THE RELAY DOES NOT ANSWER. Any failure of the +// primary address other than a signature failure falls through to the mirror, +// which is the relay's own document copied elsewhere and shares the cache +// directory, so a document the mirror serves is kept only when its version is +// not lower than the one already cached. A signature failure is its own +// statement about the primary's bytes, and a copy of the same document cannot +// vouch for them, so it is not asked; an empty mirror address turns the +// fallback off. +func refreshPoolIndex(ctx context.Context, profileDir string, cfg poolcfg.Config, keys []ed25519.PublicKey) { + poolDir := config.ProfilePath(profileDir, "pool") + if err := poolIndexPull(ctx, cfg.IndexURL, poolDir, cfg.TTL, keys); err != nil { + if errors.Is(err, pull.ErrBadSignature) || cfg.MirrorURL == "" { + if trace.Enabled() { + log.Printf("model pool: index refresh: %v", err) + } + return + } + if err := poolIndexPull(ctx, cfg.MirrorURL, poolDir, cfg.TTL, keys); err != nil && trace.Enabled() { + log.Printf("model pool: index refresh: %v", err) + } + } +} + +// poolIndexPull fetches url through the puller's own cache under the config's +// TTL, and returns the fetch's error — nil when a document was read or a young +// cache answered. +func poolIndexPull(ctx context.Context, url, poolDir string, ttl time.Duration, keys []ed25519.PublicKey) error { + puller := &pull.Puller{ + URL: url, + Keys: keys, + CacheDir: poolDir, + TTL: ttl, + Budget: pull.DefaultBudget, + } + _, err := puller.Pull(ctx) + return err +} + +// wirePoolIndex seats this process's pool readings beside its catalog: it +// hands the index reader to seat resolution (config.AutoIndex), the install's +// own judged scores beside them (config.AutoOwnCells, read off the own sheet +// under the pool directory), and starts the refresh beside both. It is called +// once at start-up, from the same places config.AutoModels is set, so a tier +// row that says `auto` resolves against a prior on every door that resolves a +// seat. +func wirePoolIndex(profileDir string) { + cfg := config.ModelPoolAt(profileDir) + config.AutoIndex = poolIndexFor(profileDir, cfg, time.Now) + config.AutoOwnCells = poolOwnCellsFor(profileDir, cfg) + startPoolIndexRefresh(context.Background(), profileDir, cfg, poolTrustedKeys(cfg)) + // The rows a previous run judged and could not hand over leave at once, + // on their own goroutine behind the same guard the refresh uses: a + // start-up errand, bounded by its own budget, and never on the run's + // path. + if cfg.CanSend() { + poolRefreshGo("pool/push", func() { poolPush(context.Background(), profileDir, cfg, poolPushBudget) }) + } +} + +// poolOwnCellsFor builds this process's one reader of the install's own judged +// scores. THE READ IS START-UP WORK AND NOTHING ON A RUN'S PATH, the same +// posture the index reader keeps: the own sheet under the profile's pool +// directory is read and parsed here, once, and the returned function answers +// the cells already in hand, so a pick does no disk and no decode per call. +// A mode that forbids reading answers nil — the same nothing the seam reads +// before any sheet exists — and so does a sheet that does not parse, which is +// said under the debug record's switch and read as absent rather than fatal: +// an own sheet is the install's own evidence, and a broken one is a loss, not +// a fault a pick should stop for. +func poolOwnCellsFor(profileDir string, cfg poolcfg.Config) func() []crewpick.Cell { + if !cfg.CanRead() { + return nil + } + sheet, err := record.LoadSheet(record.OwnSheetPath(config.ProfilePath(profileDir, "pool"))) + if err != nil { + if trace.Enabled() { + log.Printf("model pool: own sheet: %v", err) + } + return nil + } + cells := record.Cells(sheet) + return func() []crewpick.Cell { return cells } +} diff --git a/cmd/codeaf/poolinstall.go b/cmd/codeaf/poolinstall.go new file mode 100644 index 0000000000..6c1a1db32c --- /dev/null +++ b/cmd/codeaf/poolinstall.go @@ -0,0 +1,109 @@ +// The install's own identity at the relay: one nonce, drawn once and kept. +// +// Every batch the outbox sends to the relay rides the header +// X-Codeaf-Install with this word, so the relay can tell one install's +// measurements from another's without ever learning who the install is: the +// nonce is 32 random hex characters, held in a file only this install writes, +// and nothing else about the machine travels with it. The file is replaced, +// not repaired, when it stops being that word — a mangled file costs the +// install its old identity and nothing else. +package main + +import ( + "context" + cryptorand "crypto/rand" + "encoding/hex" + "log" + "os" + "path/filepath" + "regexp" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" + "github.com/Agent-Field/codeaf/internal/trace" +) + +// installFile is the one file an install's nonce is kept in, under the pool +// directory, beside the outbox and the own sheet. +const installFile = "install" + +// nonceLen is 16 random bytes, spelled as 32 lowercase hex — the shape the +// relay's header check asks for. +const ( + nonceLen = 16 + nonceHexLen = 32 + installDirMode = 0700 + installFileMode = 0600 +) + +// nonceWord is the only shape installNonce accepts back off the disk. +var nonceWord = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// poolPushBudget is the bound one push is given, at start-up and after a +// judged run alike: an errand nobody is waiting for gets one bound, the same +// bound at both doors so a slow relay cannot hold either open longer. +const poolPushBudget = 5 * time.Second + +// installNonce answers this install's nonce: the word in /install, +// drawn from the system's randomness and written there the first time, mode +// 0600 in a directory of mode 0700. A file that is missing, or one that does +// not hold the one accepted shape, is replaced the same way, so a mangled +// file cannot wedge an install out of sending. +func installNonce(poolDir string) (string, error) { + path := filepath.Join(poolDir, installFile) + if raw, err := os.ReadFile(path); err == nil { + if word := string(raw); nonceWord.MatchString(word) { + return word, nil + } + } else if !os.IsNotExist(err) { + return "", err + } + raw := make([]byte, nonceLen) + if _, err := cryptorand.Read(raw); err != nil { + return "", err + } + nonce := hex.EncodeToString(raw) + if err := os.MkdirAll(poolDir, installDirMode); err != nil { + return "", err + } + if err := os.WriteFile(path, []byte(nonce), installFileMode); err != nil { + return "", err + } + return nonce, nil +} + +// poolPush hands the outbox's pending rows to the relay: the rows are the +// install's judged scores, and the relay wants the install's nonce beside +// them. Quiet by design, for poolindex.go's reason — a relay that does not +// answer yet, or a nonce that could not be drawn, is an ordinary state for an +// errand nothing waits on, so the error is said only under the debug record's +// switch and never at a person. The rows a batch did not reach stay pending, +// where the next judged run and the next start-up try them again. +func poolPush(ctx context.Context, profileDir string, cfg poolcfg.Config, budget time.Duration) { + if !cfg.CanSend() { + return + } + poolDir := config.ProfilePath(profileDir, "pool") + box, err := outbox.Open(outboxPath(poolDir)) + if err != nil { + if trace.Enabled() { + log.Printf("model pool: outbox: %v", err) + } + return + } + defer box.Close() + nonce, err := installNonce(poolDir) + if err != nil { + if trace.Enabled() { + log.Printf("model pool: install nonce: %v", err) + } + return + } + box.Install = nonce + box.Budget = budget + if _, err := box.Send(ctx, cfg.SubmitURL); err != nil && trace.Enabled() { + log.Printf("model pool: send: %v", err) + } +} diff --git a/cmd/codeaf/poolinstall_test.go b/cmd/codeaf/poolinstall_test.go new file mode 100644 index 0000000000..1b36ff06e2 --- /dev/null +++ b/cmd/codeaf/poolinstall_test.go @@ -0,0 +1,216 @@ +package main + +// The install's nonce and the push that rides it: what the file holds, what +// replaces a mangled one, and what one push does to the outbox at a relay +// that answers and at one that does not. + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" +) + +var hexWord = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// The nonce is drawn once and kept: a second call in the same pool directory +// reads the same file and answers the same 32 lowercase hex characters, and +// the file sits at mode 0600 in a directory of mode 0700. +func TestInstallNonceDrawsOnceAndKeepsTheWord(t *testing.T) { + dir := t.TempDir() + poolDir := filepath.Join(dir, "pool") + + first, err := installNonce(poolDir) + if err != nil { + t.Fatalf("installNonce: %v", err) + } + if len(first) != 32 || !hexWord.MatchString(first) { + t.Fatalf("the nonce is %q, want 32 lowercase hex characters", first) + } + second, err := installNonce(poolDir) + if err != nil { + t.Fatalf("installNonce again: %v", err) + } + if second != first { + t.Fatalf("the second draw answered %q, want the kept %q", second, first) + } + + info, err := os.Stat(filepath.Join(poolDir, "install")) + if err != nil { + t.Fatalf("the nonce file: %v", err) + } + if info.Mode().Perm() != 0600 { + t.Fatalf("the nonce file is %v, want 0600", info.Mode().Perm()) + } + dirInfo, err := os.Stat(poolDir) + if err != nil { + t.Fatalf("the pool directory: %v", err) + } + if dirInfo.Mode().Perm() != 0700 { + t.Fatalf("the pool directory is %v, want 0700", dirInfo.Mode().Perm()) + } + + // A nonce drawn here is not the one another install drew: two pool + // directories hold two words. + other, err := installNonce(filepath.Join(t.TempDir(), "pool")) + if err != nil { + t.Fatalf("installNonce elsewhere: %v", err) + } + if other == first { + t.Fatal("two installs drew the same nonce") + } +} + +// A file that does not hold the one accepted shape is replaced the way a +// missing one is: a new draw, written over it. Whatever mangled the file — a +// half write, an older shape — the install is sending again afterwards. +func TestInstallNonceReplacesAMangledFile(t *testing.T) { + for name, mangled := range map[string]string{ + "not hex": "not a nonce at all", + "too short": "abcd1234", + "upper case": strings.ToUpper("0123456789abcdef0123456789abcdef"), + "the empty file": "", + "right length, junk": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + } { + dir := filepath.Join(t.TempDir(), "pool") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "install"), []byte(mangled), 0o600); err != nil { + t.Fatal(err) + } + got, err := installNonce(dir) + if err != nil { + t.Fatalf("%s: installNonce: %v", name, err) + } + if !hexWord.MatchString(got) { + t.Fatalf("%s: the mangled file was not replaced: %q", name, got) + } + if back, err := installNonce(dir); err != nil || back != got { + t.Fatalf("%s: the replacement was not kept: %q, %v", name, back, err) + } + } +} + +// A pool directory that cannot be created is an error, not a silence. +func TestInstallNonceRefusesAPlaceItCannotWrite(t *testing.T) { + file := filepath.Join(t.TempDir(), "plain") + if err := os.WriteFile(file, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := installNonce(filepath.Join(file, "pool")); err == nil { + t.Fatal("a nonce was drawn over a file") + } +} + +// ── THE PUSH ──────────────────────────────────────────────────────────────── + +// A push is the outbox's pending rows handed to the relay under the install's +// nonce: one POST, the header carrying the word the nonce file holds, and +// every row it reached marked sent in the file. +func TestPoolPushPostsPendingRowsUnderTheInstallNonce(t *testing.T) { + var installs []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + installs = append(installs, r.Header.Get("X-Codeaf-Install")) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + dir := seedOutbox(t) + submit := oneEnv("CODEAF_MODEL_POOL_SUBMIT_URL", srv.URL) + cfg := poolcfg.Resolve("", "", submit) + poolPush(context.Background(), dir, cfg, 2*time.Second) + + if len(installs) != 1 { + t.Fatalf("the relay was asked %d times, want once: %v", len(installs), installs) + } + if !hexWord.MatchString(installs[0]) { + t.Fatalf("the batch rode with X-Codeaf-Install %q, want 32 lowercase hex", installs[0]) + } + nonce, err := installNonce(configuredPoolDir(dir)) + if err != nil { + t.Fatalf("the nonce file: %v", err) + } + if installs[0] != nonce { + t.Fatalf("the header carried %q, want the word the nonce file holds %q", installs[0], nonce) + } + + // Every row the relay took is marked sent: the same outbox, reopened, + // holds nothing pending. + box, err := outboxOpenForTest(dir) + if err != nil { + t.Fatalf("the outbox: %v", err) + } + defer box.Close() + if got := len(box.Pending()); got != 0 { + t.Fatalf("%d row(s) still pending after a delivered push", got) + } +} + +// A relay that does not answer is an ordinary state: the push says nothing, +// prints nothing, and every row stays pending where the next push finds it. +func TestPoolPushLeavesTheRowsPendingWhenTheRelayDoesNotAnswer(t *testing.T) { + dir := seedOutbox(t) + // A dead loopback port: the address the relay would have answered at, if + // the relay were there. + submit := oneEnv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/v1/rows") + cfg := poolcfg.Resolve("", "", submit) + + said, err := captureStdout(t, func() error { + poolPush(context.Background(), dir, cfg, time.Second) + return nil + }) + if err != nil { + t.Fatalf("the push: %v", err) + } + if said != "" { + t.Fatalf("a push nothing waits for said %q", said) + } + + box, err := outboxOpenForTest(dir) + if err != nil { + t.Fatalf("the outbox: %v", err) + } + defer box.Close() + if got := len(box.Pending()); got != 2 { + t.Fatalf("%d row(s) pending after an unanswered push, want the 2 seeded", got) + } +} + +// A mode that does not send pushes nothing at all: no outbox opened, no +// nonce drawn, no file written under the pool directory. +func TestPoolPushDoesNothingWhenTheModeDoesNotSend(t *testing.T) { + dir := t.TempDir() + poolDir := filepath.Join(dir, "pool") + if err := os.MkdirAll(poolDir, 0o700); err != nil { + t.Fatal(err) + } + cfg := poolcfg.Resolve("read", "", oneEnv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/v1/rows")) + poolPush(context.Background(), dir, cfg, time.Second) + if _, err := os.Stat(filepath.Join(poolDir, "install")); !os.IsNotExist(err) { + t.Fatal("a read-only pool drew a nonce") + } + if _, err := os.Stat(filepath.Join(poolDir, "outbox.jsonl")); !os.IsNotExist(err) { + t.Fatal("a read-only pool opened the outbox") + } +} + +// configuredPoolDir is the pool directory a push under profileDir writes +// into, the same reading the push itself makes. +func configuredPoolDir(profileDir string) string { + return filepath.Join(profileDir, "pool") +} + +// outboxOpenForTest opens the outbox file the way a reading test reads it +// back — the same path poolPush writes through config.ProfilePath. +func outboxOpenForTest(dir string) (*outbox.Outbox, error) { + return outbox.Open(filepath.Join(dir, "pool", "outbox.jsonl")) +} diff --git a/cmd/codeaf/poolrecord.go b/cmd/codeaf/poolrecord.go new file mode 100644 index 0000000000..852c882527 --- /dev/null +++ b/cmd/codeaf/poolrecord.go @@ -0,0 +1,340 @@ +// The Model Pool's judge, wired to the chat door's landings. +// +// When a task lands, a model outside the crew scores each seat the work ran +// on. The scores go into the install's own sheet under the pool directory — +// the one the crew picker already reads beside the index — and, when the +// pool's mode allows sending, into the outbox beside it. The pure halves live +// in internal/pool/judge and internal/pool/record; this file owns the two +// things they cannot: the seam in the engine's landing (session.Config's +// TaskLanded) and the one provider call the judge's questions ride, billed to +// the judge's own seat. +// +// Quiet by design, for poolindex.go's reason: a landing that nobody could +// score is an ordinary state — a pool switched off, a catalog with nothing +// left to pick, a model that did not answer — and nothing the person is +// reading should move for it. Every error here is said only under the debug +// record's switch. +package main + +import ( + "context" + "encoding/json" + "errors" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/crewpick" + "github.com/Agent-Field/codeaf/internal/guard" + "github.com/Agent-Field/codeaf/internal/lane" + "github.com/Agent-Field/codeaf/internal/pool/judge" + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/pool/record" + "github.com/Agent-Field/codeaf/internal/provider" + "github.com/Agent-Field/codeaf/internal/session" + "github.com/Agent-Field/codeaf/internal/trace" +) + +// judgeTimeout bounds ONE seat's question, and the landing as a whole gets one +// share per seat and one to spare, so a slow model cannot hold a landing open +// forever and a slow first answer does not eat the second question's time: a +// reasoning judge took 64 s over the worker and the checker's question was +// then cut at the landing's 90 s, and the checker seat went unscored. It is a +// bound on an errand nobody is waiting for, not on work the person asked for, +// and it is generous on purpose — the questions are one call each. +const judgeTimeout = 90 * time.Second + +// judgeTries bounds how many candidates one landing is put through. A judge +// that answers no seat at all — every question 429ed, refused or timed out — is +// the judge's own failure rather than a verdict on the run, so the next +// candidate is asked; the cap is what keeps a landing no judge will answer from +// holding the pool's goroutine past a few candidates' worth of the bound above. +const judgeTries = 3 + +// poolJudgeHook builds the session's landing reader. Nil is off, for the pool +// index's reason: a mode that forbids reading runs no judge and writes +// nothing, and the door then hands the engine no hook at all. +func poolJudgeHook(settings config.Config, profileDir, workspace string, models func() []catalog.Model, ask func(model string) judge.Ask, now func() time.Time) func(session.TaskLanding) { + if !config.ModelPoolAt(profileDir).CanRead() { + return nil + } + return func(landing session.TaskLanding) { + defer guard.Recover("pool/judge") + poolJudgeLanding(settings, profileDir, models, ask, now, landing) + } +} + +// judgeLast is the small record the hook leaves after every landing it +// handles, for the one reading that would otherwise need --debug: what the +// last judge did, and what became of it. Judge names the model that answered +// and Scored the seats it scored on a success; on a failure Judge is empty, +// Tried the candidates in the order they were asked and Reason the last +// error's one line. A landing the hook declines before asking carries the +// decline itself as the reason. +type judgeLast struct { + At time.Time `json:"at"` + Task uint64 `json:"task"` + Judge string `json:"judge"` + Tried []string `json:"tried"` + Seats []string `json:"seats"` + Scored []string `json:"scored"` + Reason string `json:"reason"` +} + +// errNoJudgeCandidate is the decline a landing with nothing to ask says: the +// one sentence a person reads when the record holds no candidates at all. +const errNoJudgeCandidate = "no judge: every candidate is in the crew, unpriced, free, or below the floor" + +// writeJudgeLast leaves the record under the pool directory, mode 0600 like +// the sheet beside it. It never panics and its errors are debug-only, for the +// hook's own reason: a record that could not be written must not cost a +// landing its scores, and nobody reading the chat surface moves for it. +func writeJudgeLast(poolDir string, last judgeLast) { + defer guard.Recover("pool/judge-last") + data, err := json.Marshal(last) + if err != nil { + if trace.Enabled() { + log.Printf("model pool: judge-last: %v", err) + } + return + } + if err := os.MkdirAll(poolDir, 0o700); err != nil { + if trace.Enabled() { + log.Printf("model pool: judge-last: %v", err) + } + return + } + if err := os.WriteFile(filepath.Join(poolDir, "judge-last.json"), data, 0o600); err != nil && trace.Enabled() { + log.Printf("model pool: judge-last: %v", err) + } +} + +// poolJudgeLanding scores one landed task and records what came back. The +// order is the sheet's own law: every score is observed before the rows are +// appended, the sheet is saved once, and the picker's own-cells seam is +// repointed at the new cells so the very next pick in this process reads them. +func poolJudgeLanding(settings config.Config, profileDir string, models func() []catalog.Model, ask func(model string) judge.Ask, now func() time.Time, landing session.TaskLanding) { + pool := config.ModelPoolAt(profileDir) + poolDir := config.ProfilePath(profileDir, "pool") + seats := map[judge.Role]string{ + judge.RoleWorker: landing.Worker, + } + if landing.High != "" { + seats[judge.RoleHigh] = landing.High + } + candidates := judge.Candidates(models(), seats, judge.DefaultFloor) + // The seats are said in the crew's own order — the worker, then the high + // seat when the run held one — so the record reads the way the crew ran. + held := []string{landing.Worker} + if landing.High != "" { + held = append(held, landing.High) + } + if len(candidates) == 0 { + if trace.Enabled() { + log.Printf("model pool: no judge to ask about task %d", landing.ID) + } + writeJudgeLast(poolDir, judgeLast{At: now(), Task: landing.ID, Seats: held, Reason: errNoJudgeCandidate}) + return + } + if len(candidates) > judgeTries { + candidates = candidates[:judgeTries] + } + rec := judge.Record{ + Brief: landing.Brief, + Deliverable: landing.Deliverable, + Report: landing.Report, + Claim: landing.Claim, + Ending: landing.Ending, + Files: landing.Wrote, + Changed: landing.Changed, + Checks: landing.Checks, + Seats: seats, + CostUSD: landing.CostUSD, + } + // A landing is news, not a turn: nobody is waiting on the answer, and the + // one thing this context owes anybody is a bound on how long it holds the + // pool's own goroutine. Each candidate gets that whole bound afresh, so the + // first judge's share is spent by the first judge and the next starts with a + // full one rather than the first's leftovers. + var judgeID string + var scores []judge.Score + var err error + for _, candidate := range candidates { + rec.Judge = candidate + ctx, cancel := context.WithTimeout(context.Background(), judgeTimeout*time.Duration(len(seats)+1)) + one := ask(candidate) + perSeat := func(ctx context.Context, system, user string) (string, error) { + qctx, cancel := context.WithTimeout(ctx, judgeTimeout) + defer cancel() + return one(qctx, system, user) + } + scores, err = judge.Judge(ctx, perSeat, rec) + cancel() + if len(scores) > 0 { + judgeID = candidate + break + } + // No seat at all came back: that is the judge's own failure — a 429, a + // timeout, a refusal — and not a verdict on the run, so its reason is + // said here and the next candidate is asked. + if trace.Enabled() { + log.Printf("model pool: judge %s scored no seat of task %d: %v", candidate, landing.ID, err) + } + } + // What the judge did is said before anything else is recorded, so the + // record stands even when the sheet or the outbox refuses it: a failure + // names the candidates in the order they were asked and the last error's + // one line; a success names the judge that answered and the seats it + // scored, with no reason to say. + if len(scores) == 0 { + reason := "no judge answered" + if err != nil { + reason = oneLine(err.Error()) + } + writeJudgeLast(poolDir, judgeLast{At: now(), Task: landing.ID, Tried: candidates, Seats: held, Reason: reason}) + if trace.Enabled() { + log.Printf("model pool: no judge scored task %d", landing.ID) + } + return + } + scored := make([]string, 0, len(scores)) + for _, seat := range scores { + scored = append(scored, seat.Model) + } + writeJudgeLast(poolDir, judgeLast{At: now(), Task: landing.ID, Judge: judgeID, Seats: held, Scored: scored}) + // The scores obtained are recorded even when the error names a seat: a + // judge that scored the worker but not the high seat scored the worker, + // and a seat the call failed on is the judge's evidence of that model too. + if err != nil && trace.Enabled() { + log.Printf("model pool: judging task %d: %v", landing.ID, err) + } + sheet, err := record.LoadSheet(record.OwnSheetPath(poolDir)) + if err != nil { + if trace.Enabled() { + log.Printf("model pool: own sheet: %v", err) + } + return + } + recorder := &record.Recorder{Sheet: sheet} + if pool.CanSend() { + ob, err := outbox.Open(outboxPath(poolDir)) + if err != nil { + if trace.Enabled() { + log.Printf("model pool: outbox: %v", err) + } + } else { + recorder.Outbox = ob + defer ob.Close() + } + } + day := now().UTC().Format("2006-01-02") + if err := recorder.Record(scores, judgeID, "task", poolSize(landing.Tokens), day); err != nil && trace.Enabled() { + log.Printf("model pool: record: %v", err) + } + if err := record.SaveSheet(record.OwnSheetPath(poolDir), sheet); err != nil { + if trace.Enabled() { + log.Printf("model pool: own sheet: %v", err) + } + return + } + // The next pick in this process reads the new cells at once, the same way + // the picker reads them at start-up (poolindex.go's poolOwnCellsFor): a + // closing one is the install's own evidence and is never held to the + // index's min_installs. + cells := record.Cells(sheet) + config.AutoOwnCells = func() []crewpick.Cell { return cells } + // The sheet is saved, so what the recorder appended is the install's own + // evidence now; the copies waiting in the outbox leave for the relay + // here, on this hook's own goroutine (the session runs TaskLanded on + // one), under the push's own bound. + pushCtx, cancelPush := context.WithTimeout(context.Background(), poolPushBudget) + defer cancelPush() + poolPush(pushCtx, profileDir, pool, poolPushBudget) +} + +// poolSize is the day-row bucket a landing's token count answers: S under +// 200k tokens, M under a million, L past it. The size says how much work the +// bill was carrying, which is the one thing a score alone does not say. +func poolSize(tokens int) string { + switch { + case tokens < 200_000: + return "S" + case tokens < 1_000_000: + return "M" + default: + return "L" + } +} + +// outboxPath is the one path an install's measurements leave by, beside its +// own sheet under the pool directory. +func outboxPath(poolDir string) string { + return filepath.Join(poolDir, "outbox.jsonl") +} + +// poolJudgeAsk builds the maker of one judge's ask: a client per call, built +// from the model's own config, the answer read as the judge's plain string, +// and the call billed to the judge's own seat. +func poolJudgeAsk(settings config.Config, profileDir string) func(model string) judge.Ask { + return func(model string) judge.Ask { + return func(ctx context.Context, system, user string) (string, error) { + client, err := provider.NewClient(settings.ClientConfig(model)) + if err != nil { + return "", err + } + // The call says who it is for the way every errand does (chat.go's + // errandContext): the role names the seat the bill lands on, the tag + // names the rows in the model-call log. Streaming buys nothing — the + // answer is one object nobody is reading as it arrives. + ctx = provider.WithCallTag(provider.WithRole(settings.Context(ctx, "pool-judge"), lane.RoleJudge), "judge") + ctx = provider.WithoutStream(ctx) + messages := []ai.Message{ + judgeMessage("system", system), + judgeMessage("user", user), + } + response, err := client.CompleteWithMessages(ctx, messages) + if err != nil { + return "", err + } + if response == nil || strings.TrimSpace(response.Text()) == "" { + return "", errors.New("the model answered nothing") + } + recordPoolUsage(settings, model, response.Usage) + return response.Text(), nil + } + } +} + +// judgeMessage is one plain-text part in one role, the shape every +// single-question errand here builds its pair out of. +func judgeMessage(role, text string) ai.Message { + return ai.Message{Role: role, Content: []ai.ContentPart{{Type: "text", Text: text}}} +} + +// recordPoolUsage appends the call's one row to this machine's usage ledger. +// THE SEAT IS THE JUDGE'S, because the call was not a conversation's turn and +// no crew seat asked for it. The dollars are the provider's own receipt when +// one arrived, otherwise the model's published price over the tokens it +// counted — and zero when nobody published one, which reads as "nobody said", +// the same reading every other row's zero has. +func recordPoolUsage(settings config.Config, model string, used *ai.Usage) { + line := session.UsageLine{ + Seat: session.SeatJudge, + Model: model, + Calls: 1, + } + if used != nil { + line.Input, line.Output = used.PromptTokens, used.CompletionTokens + if used.Cost != nil { + line.USD = *used.Cost + } else if prompt, completion, known := settings.ClientConfig(model).ModelPrice(model); known { + line.USD = float64(used.PromptTokens)*prompt + float64(used.CompletionTokens)*completion + } + } + session.RecordUsage(session.UsageLedgerPath(), line) +} diff --git a/cmd/codeaf/poolrecord_test.go b/cmd/codeaf/poolrecord_test.go new file mode 100644 index 0000000000..bb8f5f2b2a --- /dev/null +++ b/cmd/codeaf/poolrecord_test.go @@ -0,0 +1,695 @@ +package main + +// The Model Pool's judge, at the chat door: what a landing records, what it +// costs, and what it refuses to do. +// +// Every test here runs the real hook against a temp profile — a fake ask +// standing in for the provider call, a fake catalog standing in for the +// list — and reads back the three records the brief names: the own sheet, the +// outbox, and the usage ledger. The pool's mode is pinned from the +// environment, the same word `model_pool` resolves through in production. + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/pool/judge" + "github.com/Agent-Field/codeaf/internal/pool/record" + "github.com/Agent-Field/codeaf/internal/session" +) + +// poolTestCatalog is three rows: the two models the crew held, and one model +// outside it, dear enough at its published coding index to be picked and with +// a price and "tools" published, which is what Pick asks a candidate to carry. +func poolTestCatalog() []catalog.Model { + return []catalog.Model{ + {ID: "crew/worker", PromptPrice: 1, CompletionPrice: 2, CodingIndex: 80, Parameters: []string{"tools"}}, + {ID: "crew/high", PromptPrice: 1, CompletionPrice: 2, CodingIndex: 80, Parameters: []string{"tools"}}, + {ID: "other/judge", PromptPrice: 0.5, CompletionPrice: 1, CodingIndex: 90, Parameters: []string{"tools"}}, + } +} + +// poolTestAsk is the fake ask: it records the model it was asked for, bills +// the call exactly as the real ask bills one — the provider's own receipt, a +// fifth of a cent — and answers the one JSON object a seat's answer is. +func poolTestAsk(settings config.Config, asked *[]string) func(model string) judge.Ask { + receipt := 0.0002 + return func(model string) judge.Ask { + return func(context.Context, string, string) (string, error) { + *asked = append(*asked, model) + recordPoolUsage(settings, model, &ai.Usage{PromptTokens: 10, CompletionTokens: 20, Cost: &receipt}) + return `{"score": 88, "reason": "the delivered work does what the brief asked"}`, nil + } + } +} + +// poolTestLanding is one landed task with both judged seats held. +func poolTestLanding() session.TaskLanding { + return session.TaskLanding{ + ID: 7, + State: session.TaskDone, + Brief: "read ALPHA and say what it holds", + Deliverable: "the reading", + Report: "the report the node landed with", + Wrote: []string{"alpha.md"}, + Changed: 1, + Checks: []string{"go test ./..."}, + Worker: "crew/worker", + High: "crew/high", + Tokens: 150_000, + } +} + +// restoreOwnCells puts the picker's own-cells seam back after a test that +// repointed it, so no other test in the process reads this test's sheet. +func restoreOwnCells(t *testing.T) { + t.Helper() + previous := config.AutoOwnCells + t.Cleanup(func() { config.AutoOwnCells = previous }) +} + +// TestPoolJudgeHookScoresALandedTaskIntoItsOwnSheetAndAnswersTheNewCells runs +// one landing through the real hook and reads back the whole of it: one cell +// per held seat in the install's own sheet, the seam answering those cells at +// once, the judge's questions asked of no model the crew held, and the calls +// billed to the judge's seat. +func TestPoolJudgeHookScoresALandedTaskIntoItsOwnSheetAndAnswersTheNewCells(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + var asked []string + hook := poolJudgeHook(settings, profileDir, t.TempDir(), poolTestCatalog, poolTestAsk(settings, &asked), time.Now) + if hook == nil { + t.Fatal("a pool whose mode allows reading built no hook") + } + hook(poolTestLanding()) + + sheet, err := record.LoadSheet(record.OwnSheetPath(config.ProfilePath(profileDir, "pool"))) + if err != nil { + t.Fatalf("the own sheet: %v", err) + } + cells := record.Cells(sheet) + if len(cells) != 2 { + t.Fatalf("the sheet holds %d cells, want one per held seat: %+v", len(cells), cells) + } + seen := map[string]bool{} + for _, cell := range cells { + seen[cell.Role+"/"+cell.Model] = true + if cell.N != 1 { + t.Fatalf("the %s seat's cell holds %d readings, want 1", cell.Role, cell.N) + } + } + for _, seat := range []string{"worker/crew/worker", "high/crew/high"} { + if !seen[seat] { + t.Fatalf("the sheet holds no cell for %s: %+v", seat, cells) + } + } + + if config.AutoOwnCells == nil { + t.Fatal("the picker's own-cells seam was not repointed after the landing") + } + if answered := config.AutoOwnCells(); len(answered) != 2 { + t.Fatalf("the seam answers %d cells, want the 2 the sheet now holds", len(answered)) + } + + if len(asked) != 2 || asked[0] != "other/judge" || asked[1] != "other/judge" { + t.Fatalf("the judge asked %v, want only other/judge — never a model the crew held", asked) + } + + session.FlushUsage() + rows, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatalf("the usage ledger: %v", err) + } + if len(rows) != 2 { + t.Fatalf("the ledger holds %d rows, want one per seat question", len(rows)) + } + for _, row := range rows { + if row.Seat != session.SeatJudge { + t.Fatalf("a call was billed to seat %q, want judge", row.Seat) + } + if row.Model != "other/judge" { + t.Fatalf("a call was billed to model %q", row.Model) + } + if row.Input != 10 || row.Output != 20 { + t.Fatalf("a row carries %d in / %d out", row.Input, row.Output) + } + } +} + +// TestPoolJudgeHookAppendsOutboxRowsOnlyWhenTheModeSends walks the three +// postures one after another: a mode that sends appends one row per seat, a +// mode whose submit address was emptied sends nothing, and a read-only mode +// records locally only. +func TestPoolJudgeHookAppendsOutboxRowsOnlyWhenTheModeSends(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + restoreOwnCells(t) + + profileDir := t.TempDir() + outboxFile := filepath.Join(config.ProfilePath(profileDir, "pool"), "outbox.jsonl") + settings := config.Config{} + var asked []string + models := poolTestCatalog + + hook := func() func(session.TaskLanding) { + return poolJudgeHook(settings, profileDir, t.TempDir(), models, poolTestAsk(settings, &asked), time.Now) + } + + // A pool that sends: one row per seat question, outbox beside the sheet. + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + hook()(poolTestLanding()) + data, err := os.ReadFile(outboxFile) + if err != nil { + t.Fatalf("the outbox: %v", err) + } + if lines := countLines(data); lines != 2 { + t.Fatalf("the outbox holds %d rows, want 2", lines) + } + + // A submit address emptied on purpose: the sheet is still observed, and + // nothing waits to leave. + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "") + os.Remove(outboxFile) + hook()(poolTestLanding()) + if _, err := os.Stat(outboxFile); !os.IsNotExist(err) { + t.Fatalf("a pool with no submit address wrote %s", outboxFile) + } + + // A read-only pool: the same, and the sheet is still kept. + t.Setenv("CODEAF_MODEL_POOL", "read") + hook()(poolTestLanding()) + if _, err := os.Stat(outboxFile); !os.IsNotExist(err) { + t.Fatalf("a read-only pool wrote %s", outboxFile) + } + sheet, err := record.LoadSheet(record.OwnSheetPath(config.ProfilePath(profileDir, "pool"))) + if err != nil { + t.Fatalf("the own sheet after a read-only landing: %v", err) + } + if got := len(record.Cells(sheet)); got != 2 { + t.Fatalf("the sheet holds %d cells after three landings, want 2 (one per seat, observed three times)", got) + } +} + +// TestPoolJudgeHookDoesNothingUnderModeOff is the switch: a pool whose mode +// forbids reading builds no hook at all, so a landing costs no call and +// writes nothing. +func TestPoolJudgeHookDoesNothingUnderModeOff(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "off") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + var asked []string + if hook := poolJudgeHook(settings, profileDir, t.TempDir(), poolTestCatalog, poolTestAsk(settings, &asked), time.Now); hook != nil { + t.Fatal("a pool whose mode forbids reading built a hook anyway") + } + if _, err := os.Stat(record.OwnSheetPath(config.ProfilePath(profileDir, "pool"))); !os.IsNotExist(err) { + t.Fatal("a pool whose mode forbids reading wrote an own sheet") + } + if len(asked) != 0 { + t.Fatalf("a pool whose mode forbids reading asked %v", asked) + } +} + +// TestPoolJudgeAskBillsTheJudgeSeatFromTheModelsPrice pins the two readings +// the bill can take: the provider's own receipt when one arrived, and the +// model's published price over the tokens when none did — with zero meaning +// nobody published one, which reads as "nobody said" and never as free. +func TestPoolJudgeAskBillsTheJudgeSeatFromTheModelsPrice(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + + // The receipt wins, whatever the price says. + receipt := 0.0007 + recordPoolUsage(config.Config{}, "other/judge", &ai.Usage{PromptTokens: 10, CompletionTokens: 20, Cost: &receipt}) + // No receipt and no published price: the tokens still happened, so the row + // carries them with no dollars — zero reads as "nobody said", never free. + recordPoolUsage(config.Config{}, "other/judge", &ai.Usage{PromptTokens: 10, CompletionTokens: 20}) + session.FlushUsage() + + rows, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatalf("the usage ledger: %v", err) + } + if len(rows) != 2 { + t.Fatalf("the ledger holds %d rows, want 2", len(rows)) + } + if rows[0].Seat != session.SeatJudge || rows[0].Model != "other/judge" { + t.Fatalf("the row names seat %q model %q", rows[0].Seat, rows[0].Model) + } + if rows[0].USD != receipt { + t.Fatalf("the row says $%v, want the provider's own $%v", rows[0].USD, receipt) + } + if rows[1].USD != 0 || rows[1].Input != 10 || rows[1].Output != 20 { + t.Fatalf("an unpriced row says $%v over %d/%d tokens", rows[1].USD, rows[1].Input, rows[1].Output) + } +} + +// TestPoolJudgeHookGivesEachSeatsQuestionItsOwnShareOfTheLandingTime pins +// the share itself: the landing's context spans every seat and one to spare, +// but each seat's question is wrapped in its own judgeTimeout, so the second +// question's deadline sits a full share after the moment it was asked and a +// slow first answer cannot eat it. The first answer is slow — it holds its +// question before answering — and the share is read off the deadlines, not +// waited out: judgeTimeout is a constant nobody wants a test to sit through. +func TestPoolJudgeHookGivesEachSeatsQuestionItsOwnShareOfTheLandingTime(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + const slow = 50 * time.Millisecond + var askedAt []time.Time + var deadlines []time.Time + ask := func(model string) judge.Ask { + return func(ctx context.Context, system, user string) (string, error) { + askedAt = append(askedAt, time.Now()) + deadline, ok := ctx.Deadline() + if !ok { + t.Fatal("a seat's question carries no deadline") + } + deadlines = append(deadlines, deadline) + if len(askedAt) == 1 { + // The first answer takes its time; the share it spends is the + // share the second question must not have lost. + select { + case <-time.After(slow): + case <-ctx.Done(): + return "", ctx.Err() + } + } + return `{"score": 88, "reason": "the delivered work does what the brief asked"}`, nil + } + } + hook := poolJudgeHook(settings, profileDir, t.TempDir(), poolTestCatalog, ask, time.Now) + if hook == nil { + t.Fatal("a pool whose mode allows reading built no hook") + } + hook(poolTestLanding()) + + if len(deadlines) != 2 || len(askedAt) != 2 { + t.Fatalf("the judge asked %d questions, want 2", len(deadlines)) + } + // The second question is asked after the first has answered, so its own + // share is judged from where it stands, not from the landing's start: a + // full judgeTimeout ahead of the asking, and no more than that — the + // landing's wider bound is not what the question was handed. + share := deadlines[1].Sub(askedAt[1]) + if share < judgeTimeout-time.Second { + t.Fatalf("the second question holds %v of context, want at least its own %v share", share, judgeTimeout-time.Second) + } + if share > judgeTimeout { + t.Fatalf("the second question holds %v of context, want its own %v share and not the landing's", share, judgeTimeout) + } + // The time the first answer spent moved the second deadline out by as + // much; it did not come off the second question's share. + if moved := deadlines[1].Sub(deadlines[0]); moved < slow { + t.Fatalf("the second deadline sits %v after the first, want at least the %v the first answer took", moved, slow) + } + + sheet, err := record.LoadSheet(record.OwnSheetPath(config.ProfilePath(profileDir, "pool"))) + if err != nil { + t.Fatalf("the own sheet: %v", err) + } + cells := record.Cells(sheet) + if len(cells) != 2 { + t.Fatalf("the sheet holds %d cells, want one per held seat: %+v", len(cells), cells) + } + seen := map[string]bool{} + for _, cell := range cells { + seen[cell.Role+"/"+cell.Model] = true + } + for _, seat := range []string{"worker/crew/worker", "high/crew/high"} { + if !seen[seat] { + t.Fatalf("the sheet holds no cell for %s: %+v", seat, cells) + } + } +} + +// TestPoolJudgeHookStillScoresTheSecondSeatWhenTheFirstSeatsShareRunsOut is +// the failure the share exists for: the first question's share runs out and +// the answer is the context's own error, and the second seat is scored anyway +// with a full share of its own — its share began when its question was asked, +// not when the landing did. The share running out is what a provider call +// answers once its context gives out; the ninety seconds themselves are not +// waited out here. +func TestPoolJudgeHookStillScoresTheSecondSeatWhenTheFirstSeatsShareRunsOut(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + question := 0 + var secondAskedAt, secondDeadline time.Time + ask := func(model string) judge.Ask { + return func(ctx context.Context, system, user string) (string, error) { + question++ + if question == 1 { + return "", context.DeadlineExceeded + } + secondAskedAt = time.Now() + secondDeadline, _ = ctx.Deadline() + return `{"score": 88, "reason": "the delivered work does what the brief asked"}`, nil + } + } + hook := poolJudgeHook(settings, profileDir, t.TempDir(), poolTestCatalog, ask, time.Now) + if hook == nil { + t.Fatal("a pool whose mode allows reading built no hook") + } + hook(poolTestLanding()) + + if question != 2 { + t.Fatalf("the judge asked %d questions, want the second seat asked after the first's share ran out", question) + } + if share := secondDeadline.Sub(secondAskedAt); share < judgeTimeout-time.Second { + t.Fatalf("the second question holds %v of context, want at least its own %v share", share, judgeTimeout-time.Second) + } + + sheet, err := record.LoadSheet(record.OwnSheetPath(config.ProfilePath(profileDir, "pool"))) + if err != nil { + t.Fatalf("the own sheet: %v", err) + } + cells := record.Cells(sheet) + if len(cells) != 1 || cells[0].Role != "high" || cells[0].Model != "crew/high" { + t.Fatalf("the sheet holds %+v, want the high seat scored after the worker's share ran out", cells) + } + if cells[0].N != 1 { + t.Fatalf("the high seat's cell holds %d readings, want 1", cells[0].N) + } +} + +// decodeJudgeLast reads the hook's own record of the last landing it judged. +func decodeJudgeLast(t *testing.T, profileDir string) judgeLast { + t.Helper() + data, err := os.ReadFile(filepath.Join(config.ProfilePath(profileDir, "pool"), "judge-last.json")) + if err != nil { + t.Fatalf("the judge-last record: %v", err) + } + var last judgeLast + if err := json.Unmarshal(data, &last); err != nil { + t.Fatalf("the judge-last record does not parse: %v", err) + } + return last +} + +// The hook names the judge that answered and the seats it scored in the small +// record beside the sheet, so a judge failure reads without --debug. +func TestPoolJudgeHookRecordsTheJudgeThatScoredTheLanding(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + var asked []string + hook := poolJudgeHook(settings, profileDir, t.TempDir(), poolTestCatalog, poolTestAsk(settings, &asked), time.Now) + hook(poolTestLanding()) + + last := decodeJudgeLast(t, profileDir) + if last.Judge != "other/judge" { + t.Fatalf("the record names judge %q, want the one that answered", last.Judge) + } + if last.Reason != "" { + t.Fatalf("a scored landing carries a reason: %q", last.Reason) + } + if strings.Join(last.Scored, ", ") != "crew/worker, crew/high" { + t.Fatalf("the record scored %v, want both held seats", last.Scored) + } + if strings.Join(last.Seats, ", ") != "crew/worker, crew/high" { + t.Fatalf("the record holds seats %v, want the crew's own order", last.Seats) + } + if last.Task != 7 { + t.Fatalf("the record names task %d, want 7", last.Task) + } + if last.At.IsZero() { + t.Fatal("the record carries no moment") + } + info, err := os.Stat(filepath.Join(config.ProfilePath(profileDir, "pool"), "judge-last.json")) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("the record's mode is %v with err %v, want 0600", info, err) + } +} + +// When every candidate fails, the record says which were asked and the last +// error's one line — the reading a person needs when no score landed. +func TestPoolJudgeHookRecordsTheCandidatesAndReasonWhenEveryJudgeFails(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + ask := func(model string) judge.Ask { + return func(context.Context, string, string) (string, error) { + return "", errors.New("the model answered with a 429") + } + } + hook := poolJudgeHook(settings, profileDir, t.TempDir(), poolTwoJudgeCatalog, ask, time.Now) + hook(poolTestLanding()) + + last := decodeJudgeLast(t, profileDir) + if last.Judge != "" { + t.Fatalf("a landing no judge scored names judge %q", last.Judge) + } + if strings.Join(last.Tried, ", ") != "other/flaky, other/steady" { + t.Fatalf("the record tried %v, want both candidates in order", last.Tried) + } + if !strings.Contains(last.Reason, "the model answered with a 429") || strings.Contains(last.Reason, "\n") { + t.Fatalf("the record's reason is %q, want the last error's one line", last.Reason) + } +} + +// A landing the hook declines before asking — nothing outside the crew to +// pick — is written too, with the decline itself as the reason. +func TestPoolJudgeHookRecordsTheDeclineWhenThereIsNoCandidate(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + var asked []string + crewOnly := func() []catalog.Model { + return poolTestCatalog()[:2] + } + hook := poolJudgeHook(settings, profileDir, t.TempDir(), crewOnly, poolTestAsk(settings, &asked), time.Now) + hook(poolTestLanding()) + + if len(asked) != 0 { + t.Fatalf("a declined landing asked %v", asked) + } + last := decodeJudgeLast(t, profileDir) + if last.Judge != "" || len(last.Tried) != 0 { + t.Fatalf("a declined landing names judge %q tried %v", last.Judge, last.Tried) + } + if last.Reason != "no judge: every candidate is in the crew, unpriced, free, or below the floor" { + t.Fatalf("the record's reason is %q, want the decline's own sentence", last.Reason) + } +} + +// countLines counts the newlines a row-per-line file holds. +func countLines(data []byte) int { + lines := 0 + for _, b := range data { + if b == '\n' { + lines++ + } + } + return lines +} + +// poolTwoJudgeCatalog is poolTestCatalog plus a second judge dearer than the +// first, so a landing has a candidate to move to when the cheapest answers +// nothing at all. +func poolTwoJudgeCatalog() []catalog.Model { + return []catalog.Model{ + {ID: "crew/worker", PromptPrice: 1, CompletionPrice: 2, CodingIndex: 80, Parameters: []string{"tools"}}, + {ID: "crew/high", PromptPrice: 1, CompletionPrice: 2, CodingIndex: 80, Parameters: []string{"tools"}}, + {ID: "other/flaky", PromptPrice: 0.4, CompletionPrice: 0.4, CodingIndex: 90, Parameters: []string{"tools"}}, + {ID: "other/steady", PromptPrice: 0.6, CompletionPrice: 0.6, CodingIndex: 90, Parameters: []string{"tools"}}, + } +} + +// poolTwoJudgeAsk answers as poolTestAsk does, except for the cheapest judge, +// which answers every question with an error and no score — the rate-limited +// row's own failure, and the one a landing must move past. +func poolTwoJudgeAsk(settings config.Config, asked *[]string) func(model string) judge.Ask { + receipt := 0.0002 + return func(model string) judge.Ask { + return func(context.Context, string, string) (string, error) { + *asked = append(*asked, model) + if model == "other/flaky" { + return "", errors.New("the model answered with a 429") + } + recordPoolUsage(settings, model, &ai.Usage{PromptTokens: 10, CompletionTokens: 20, Cost: &receipt}) + return `{"score": 88, "reason": "the delivered work does what the brief asked"}`, nil + } + } +} + +// decodeOutboxRows reads a row-per-line outbox back into the payloads it +// carries, skipping the marker lines that retire rows. +func decodeOutboxRows(t *testing.T, data []byte) []record.Row { + t.Helper() + var rows []record.Row + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + var envelope struct { + Payload json.RawMessage `json:"payload"` + } + if err := json.Unmarshal([]byte(line), &envelope); err != nil { + t.Fatalf("an outbox envelope: %v", err) + } + if len(envelope.Payload) == 0 { + continue + } + var row record.Row + if err := json.Unmarshal(envelope.Payload, &row); err != nil { + t.Fatalf("an outbox payload: %v", err) + } + rows = append(rows, row) + } + return rows +} + +// TestPoolJudgeHookMovesToTheNextCandidateWhenTheCheapestAnswersNothing is the +// brief's rule: a judge that answers no seat at all — a 429, a timeout, a +// refusal — is replaced by the next candidate, and the scores that land are +// recorded under the judge that actually answered, not the first one asked. +func TestPoolJudgeHookMovesToTheNextCandidateWhenTheCheapestAnswersNothing(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + var asked []string + hook := poolJudgeHook(settings, profileDir, t.TempDir(), poolTwoJudgeCatalog, poolTwoJudgeAsk(settings, &asked), time.Now) + if hook == nil { + t.Fatal("a pool whose mode allows reading built no hook") + } + hook(poolTestLanding()) + + if len(asked) == 0 || asked[0] != "other/flaky" { + t.Fatalf("the judge asked %v, want the cheapest candidate asked first", asked) + } + steady := 0 + for _, model := range asked { + if model == "other/steady" { + steady++ + } + } + if steady != 2 { + t.Fatalf("the second judge was asked %d questions, want one per held seat", steady) + } + + poolsheet, err := record.LoadSheet(record.OwnSheetPath(config.ProfilePath(profileDir, "pool"))) + if err != nil { + t.Fatalf("the own sheet: %v", err) + } + cells := record.Cells(poolsheet) + if len(cells) != 2 { + t.Fatalf("the sheet holds %d cells, want one per held seat: %+v", len(cells), cells) + } + seen := map[string]bool{} + for _, cell := range cells { + seen[cell.Role+"/"+cell.Model] = true + } + for _, seat := range []string{"worker/crew/worker", "high/crew/high"} { + if !seen[seat] { + t.Fatalf("the sheet holds no cell for %s: %+v", seat, cells) + } + } + + data, err := os.ReadFile(filepath.Join(config.ProfilePath(profileDir, "pool"), "outbox.jsonl")) + if err != nil { + t.Fatalf("the outbox: %v", err) + } + rows := decodeOutboxRows(t, data) + if len(rows) != 2 { + t.Fatalf("the outbox holds %d rows, want one per held seat", len(rows)) + } + for _, row := range rows { + if row.Judge != "other/steady" { + t.Fatalf("a row names judge %q, want other/steady — never the candidate that answered nothing", row.Judge) + } + } + + session.FlushUsage() + usage, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatalf("the usage ledger: %v", err) + } + if len(usage) != 2 { + t.Fatalf("the ledger holds %d rows, want one per seat question the answering judge took", len(usage)) + } + for _, row := range usage { + if row.Seat != session.SeatJudge || row.Model != "other/steady" { + t.Fatalf("a call names seat %q model %q, want the answering judge's own seat", row.Seat, row.Model) + } + } +} + +// TestPoolJudgeHookWritesNothingWhenEveryCandidateAnswersNothing is the other +// end: when no candidate returns a single score, the landing is left unrecorded +// and the hook returns without a panic. +func TestPoolJudgeHookWritesNothingWhenEveryCandidateAnswersNothing(t *testing.T) { + t.Setenv("CODEAF_HOME", t.TempDir()) + t.Setenv("CODEAF_MODEL_POOL", "on") + t.Setenv("CODEAF_MODEL_POOL_SUBMIT_URL", "http://127.0.0.1:1/submit") + restoreOwnCells(t) + + profileDir := t.TempDir() + settings := config.Config{} + var asked []string + ask := func(model string) judge.Ask { + return func(context.Context, string, string) (string, error) { + asked = append(asked, model) + return "", errors.New("the model answered with a 429") + } + } + hook := poolJudgeHook(settings, profileDir, t.TempDir(), poolTwoJudgeCatalog, ask, time.Now) + if hook == nil { + t.Fatal("a pool whose mode allows reading built no hook") + } + hook(poolTestLanding()) + + if len(asked) != 4 { + t.Fatalf("the judge asked %d questions, want both candidates asked for both seats", len(asked)) + } + if _, err := os.Stat(record.OwnSheetPath(config.ProfilePath(profileDir, "pool"))); !os.IsNotExist(err) { + t.Fatal("a landing no judge could score wrote an own sheet") + } + if _, err := os.Stat(filepath.Join(config.ProfilePath(profileDir, "pool"), "outbox.jsonl")); !os.IsNotExist(err) { + t.Fatal("a landing no judge could score wrote an outbox") + } + session.FlushUsage() + usage, err := session.ReadUsage(session.UsageLedgerPath(), time.Time{}) + if err != nil { + t.Fatalf("the usage ledger: %v", err) + } + if len(usage) != 0 { + t.Fatalf("a landing no judge could score billed %d rows", len(usage)) + } +} diff --git a/cmd/codeaf/run.go b/cmd/codeaf/run.go index e56d7644d8..c119220d4d 100644 --- a/cmd/codeaf/run.go +++ b/cmd/codeaf/run.go @@ -190,6 +190,7 @@ func runGraph(name string, args []string) error { if err != nil { return err } + useAutoSeats(settings) seats := config.ResolveSeats(settings.ProfileDir, *model, *planModel) applySeats(&settings, seats) // A graph may be loaded from disk and expanded again after an overrun, so diff --git a/cmd/codeaf/seats_wiring_test.go b/cmd/codeaf/seats_wiring_test.go index 033d56ef15..6e6c78ce87 100644 --- a/cmd/codeaf/seats_wiring_test.go +++ b/cmd/codeaf/seats_wiring_test.go @@ -148,7 +148,10 @@ func TestAnErrandWithNoFlagsRunsTheProfilesCrewAndSaysSo(t *testing.T) { t.Fatalf("the errand did not settle cleanly: %v\nstderr:\n%s", err, stderr.String()) } - work := config.TierModelAt(script.dir, config.ModelTierLow) + // THE WORK SEAT IS THE WORKER ROW, which is what [config.ResolveSeats] fills + // it from. It used to be read off the small-work row here and pass anyway, + // because the frugal preset happened to put one id on both. + work := config.TierModelAt(script.dir, config.ModelTierWorker) plan := config.TierModelAt(script.dir, config.ModelTierMastermind) for _, model := range []string{work, plan} { found := false diff --git a/cmd/codeaf/seatsrows_test.go b/cmd/codeaf/seatsrows_test.go new file mode 100644 index 0000000000..b635ec0816 --- /dev/null +++ b/cmd/codeaf/seatsrows_test.go @@ -0,0 +1,308 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/config" +) + +// THE HEADLESS DOORS WAIT FOR THE ROWS A PICK NEEDS. +// +// A pick taken off the table, and a tier row that says auto, are answered from +// the rows the process holds (config.AutoModels) — and before the doors waited, +// every cold start read that seam while the warm was still in flight, so the +// seat fell to the family's table row on a machine whose catalog was sitting in +// its own cache file. The chat surface never met the defect because its picks +// happen after the warm has landed. These tests run the doors' own two calls — +// useAutoSeats and config.ResolveSeats, the same sequence `codeaf exec` and +// `codeaf do` climb — against a profile whose catalog is cached, absent and +// unpickable, and read the rung word off the receipt either way. + +// seatPickRows is the small catalog a test cache holds: three priced rows, +// spread across the bill and across three vendors — a crew's worker and its +// careful seat may not share a vendor, and a candidate below 80% of the seat's +// best falls off its shortlist — so the rows sit close in quality, far apart in +// price, and each able to sit every seat (a window of 200k, tools, images). +// None of them is a family's table id. +func seatPickRows() []catalog.Model { + return []catalog.Model{ + {ID: "north/penny", Name: "Penny", ContextLength: 262144, + PromptPrice: 0.00001, CompletionPrice: 0.00002, CacheReadPrice: 0.000001, + IntelligenceIndex: 85, CodingIndex: 85, AgenticIndex: 85, + InputModalities: []string{"text", "image"}, OutputModalities: []string{"text"}, Parameters: []string{"tools"}}, + {ID: "south/pound", Name: "Pound", ContextLength: 262144, + PromptPrice: 0.002, CompletionPrice: 0.004, CacheReadPrice: 0.0002, + IntelligenceIndex: 86, CodingIndex: 86, AgenticIndex: 86, + InputModalities: []string{"text", "image"}, OutputModalities: []string{"text"}, Parameters: []string{"tools"}}, + {ID: "east/crown", Name: "Crown", ContextLength: 262144, + PromptPrice: 0.05, CompletionPrice: 0.1, CacheReadPrice: 0.005, + IntelligenceIndex: 100, CodingIndex: 100, AgenticIndex: 100, + InputModalities: []string{"text", "image"}, OutputModalities: []string{"text"}, Parameters: []string{"tools"}}, + } +} + +// blockedCatalogServer is a stand-in base whose /models fetch never returns: +// the cold cache behind a slow network, reduced to the one fact that matters — +// the warm does not land within any bound a test is willing to spend. The +// handler is let go exactly once, because the server refuses to close behind a +// wedged request. +func blockedCatalogServer(t *testing.T) string { + t.Helper() + release := make(chan struct{}) + var once sync.Once + letGo := func() { once.Do(func() { close(release) }) } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + t.Cleanup(server.Close) + t.Cleanup(letGo) + return server.URL +} + +// seatTheTestCatalog replaces the process's memoised catalog with a fresh one +// and restores the seams it seats when the test ends. The once is inside the +// accessor, so tests in one process would otherwise share a catalog pointed at +// whichever of them called first — which is the memoising this file is not +// about. +func seatTheTestCatalog(t *testing.T) { + t.Helper() + // The pool's refresh reaches the relay over the network and writes its + // own cache into the profile as it lands — exactly the late write a + // TempDir cleanup races. These tests read no measurements, so the pool is + // off for them: the same off nothing a row saying auto can read, which is + // the answer the seam already holds when there is no index at all. + t.Setenv("CODEAF_MODEL_POOL", "off") + previous := sharedCatalog + models, index, ownCells := config.AutoModels, config.AutoIndex, config.AutoOwnCells + sharedCatalog = newSharedCatalog() + t.Cleanup(func() { + sharedCatalog = previous + config.AutoModels, config.AutoIndex, config.AutoOwnCells = models, index, ownCells + }) +} + +// boundTheSeatWait shortens [autoSeatRowsBound] for one test and restores it, +// so the bound can be watched without spending the three seconds it carries. +func boundTheSeatWait(t *testing.T, bound time.Duration) { + t.Helper() + previous := autoSeatRowsBound + autoSeatRowsBound = bound + t.Cleanup(func() { autoSeatRowsBound = previous }) +} + +// cachedRowHeld fails the test unless the seat resolved to one of the cached +// rows — the brief's "a model from the catalog rows, not the table row". +func cachedRowHeld(t *testing.T, modelID string) { + t.Helper() + for _, row := range seatPickRows() { + if row.ID == modelID { + return + } + } + t.Fatalf("the seat resolved %q, which is not one of the cached rows", modelID) +} + +// A pick off the table is computed from the cached rows, and the receipt says +// which rung answered: `computed from the catalog` under the catalog word, +// `learned` under learn. Both doors resolve through this exact pair of calls, +// and `codeaf exec` prints the work seat's line as its whole report. +func TestADoorWithAPickWaitsForTheCachedRowsAndSaysTheRung(t *testing.T) { + for _, pick := range []string{config.CrewPickCatalog, config.CrewPickLearn} { + t.Run(pick, func(t *testing.T) { + dir := t.TempDir() + if err := catalog.Remember(catalog.Options{Dir: dir}, seatPickRows()); err != nil { + t.Fatal(err) + } + if err := config.SetCrewPick(dir, pick); err != nil { + t.Fatal(err) + } + seatTheTestCatalog(t) + t.Setenv(config.ModelEnv, "") + t.Setenv(config.PlanModelEnv, "") + + useAutoSeats(config.Config{ProfileDir: dir}) + seats := config.ResolveSeats(dir, "", "") + + want := "crew balanced, computed from the catalog" + if pick == config.CrewPickLearn { + want = "crew balanced, learned" + } + for _, seat := range []config.Seat{seats.Work, seats.Plan} { + if seat.Rung() != want { + t.Fatalf("the %s seat reads %q (%s), want the %q rung", seat.Role, seat.Model, seat.Rung(), want) + } + cachedRowHeld(t, seat.Model) + } + // And not the family's table row: [config.TierModelAt] climbs the same + // ladder and would answer the computed id here too, so the table is read + // straight off the family. + table, ok := config.CrewModelsForSource(config.CrewSourceAt(dir), config.DefaultCrew) + if !ok || seats.Work.Model == table[config.ModelTierWorker] { + t.Fatalf("the work seat fell to the family's table row %q over a catalog it holds", table[config.ModelTierWorker]) + } + // And the one line the exec door prints names the rung too. + if !strings.Contains(seats.Work.Report(), want) { + t.Fatalf("the work seat's report does not name the rung:\n%s", seats.Work.Report()) + } + }) + } +} + +// A tier row that says auto is the other way a seat needs the rows, and the +// door waits for them with no pick row in the profile at all. +func TestADoorWithAnAutoRowWaitsForTheCachedRows(t *testing.T) { + dir := t.TempDir() + if err := catalog.Remember(catalog.Options{Dir: dir}, seatPickRows()); err != nil { + t.Fatal(err) + } + rows, err := json.Marshal(map[string]string{config.KeyTierWorkerModel: config.AutoValue}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(config.BudgetConfigPath(dir), rows, 0o600); err != nil { + t.Fatal(err) + } + seatTheTestCatalog(t) + t.Setenv(config.ModelEnv, "") + t.Setenv(config.PlanModelEnv, "") + + useAutoSeats(config.Config{ProfileDir: dir}) + seats := config.ResolveSeats(dir, "", "") + + if seats.Work.Rung() != "crew balanced, computed from the catalog" { + t.Fatalf("an auto row resolved to %q (%s), want the computed rung", seats.Work.Model, seats.Work.Rung()) + } + cachedRowHeld(t, seats.Work.Model) + table, ok := config.CrewModelsForSource(config.CrewSourceAt(dir), config.DefaultCrew) + if !ok || seats.Work.Model == table[config.ModelTierWorker] { + t.Fatalf("the auto row fell to the family's table row %q over a catalog it holds", table[config.ModelTierWorker]) + } +} + +// A pick left at the table reads no rows at all, and waits for nothing — not +// even for a warm this test never lets land. The bound is raised to five +// seconds so that a wrongly added wait cannot pass by timing out quietly +// first: the door must be back in under a second. +func TestADoorAtTheTablePickWaitsForNothing(t *testing.T) { + url := blockedCatalogServer(t) + dir := t.TempDir() + if err := config.ApplyCrew(dir, config.CrewBalanced); err != nil { + t.Fatal(err) + } + seatTheTestCatalog(t) + boundTheSeatWait(t, 5*time.Second) + t.Setenv(config.ModelEnv, "") + t.Setenv(config.PlanModelEnv, "") + + started := time.Now() + useAutoSeats(config.Config{ProfileDir: dir, BaseURL: url}) + if waited := time.Since(started); waited > time.Second { + t.Fatalf("a door with no pick and no auto row waited %s for rows it does not read", waited) + } + seats := config.ResolveSeats(dir, "", "") + for _, seat := range []config.Seat{seats.Work, seats.Plan} { + if seat.Rung() != "crew balanced" { + t.Fatalf("the %s seat reads %q, want the crew's own table row", seat.Role, seat.Rung()) + } + } + if seats.Work.Model != config.TierModelAt(dir, config.ModelTierWorker) { + t.Fatalf("the work seat read %q, want the table row %q", seats.Work.Model, config.TierModelAt(dir, config.ModelTierWorker)) + } +} + +// A cold cache behind a network that never answers falls to the table row +// within the bound, and the receipt says so: `table` is a rung word on purpose, +// so a run that fell says it fell. +func TestADoorOnAColdCacheFallsToTheTableRowAndSaysSo(t *testing.T) { + url := blockedCatalogServer(t) + dir := t.TempDir() + if err := config.SetCrewPick(dir, config.CrewPickCatalog); err != nil { + t.Fatal(err) + } + seatTheTestCatalog(t) + boundTheSeatWait(t, 250*time.Millisecond) + t.Setenv(config.ModelEnv, "") + t.Setenv(config.PlanModelEnv, "") + + started := time.Now() + useAutoSeats(config.Config{ProfileDir: dir, BaseURL: url}) + seats := config.ResolveSeats(dir, "", "") + if waited := time.Since(started); waited > 2*time.Second { + t.Fatalf("the door waited %s, past the bound it was given", waited) + } + if !strings.Contains(seats.Report(), "(crew balanced, table)") { + t.Fatalf("a run that fell to the table row must say so:\n%s", seats.Report()) + } + if seats.Work.Model != config.TierModelAt(dir, config.ModelTierWorker) { + t.Fatalf("the work seat read %q, want the table row %q", seats.Work.Model, config.TierModelAt(dir, config.ModelTierWorker)) + } +} + +// THE ERRAND'S RECEIPT IS THE PART A HARNESS READS. A profile with the pick +// off the table and a catalog cached beside it runs on the computed ids, and +// both places the run names its rungs — the opening stderr lines and the +// --json object — say `computed from the catalog`. +func TestAnErrandNamesTheRungTheCatalogAnswered(t *testing.T) { + script := newScriptedBrain(t) + defer script.close() + t.Setenv(config.ModelEnv, "") + t.Setenv(config.PlanModelEnv, "") + if err := config.SetCrewPick(script.dir, config.CrewPickCatalog); err != nil { + t.Fatal(err) + } + if err := catalog.Remember(catalog.Options{Dir: script.dir, BaseURL: script.server.URL}, seatPickRows()); err != nil { + t.Fatal(err) + } + seatTheTestCatalog(t) + + var mu sync.Mutex + var built []string + var stdout, stderr strings.Builder + err := doErrand(doRequest{ + task: "write the release note and include the migration steps", + timeout: 60 * time.Second, + asJSON: true, + stdout: &stdout, + stderr: &stderr, + newClient: func(settings config.Config, model string) (*liveClient, error) { + mu.Lock() + built = append(built, model) + mu.Unlock() + return script.client(settings, model) + }, + }) + if err != nil { + t.Fatalf("the errand did not settle cleanly: %v\nstderr:\n%s", err, stderr.String()) + } + + if !strings.Contains(stderr.String(), "(crew balanced, computed from the catalog)") { + t.Fatalf("the opening lines never named the computed rung:\n%s", stderr.String()) + } + var outcome headlessOutcome + if err := json.Unmarshal([]byte(stdout.String()), &outcome); err != nil { + t.Fatalf("--json did not print one object: %v\n%s", err, stdout.String()) + } + if outcome.ModelSource != "crew balanced, computed from the catalog" { + t.Fatalf("--json named the rung %q, want the computed rung", outcome.ModelSource) + } + cachedRowHeld(t, outcome.Model) + + // And the computed id is the one the run actually sat somebody in. + mu.Lock() + models := append([]string(nil), built...) + mu.Unlock() + for _, model := range models { + if model == outcome.Model { + return + } + } + t.Fatalf("no client was built on the computed seat %q; the run used %v", outcome.Model, models) +} diff --git a/cmd/codeaf/subharness.go b/cmd/codeaf/subharness.go index 170529bed5..74c573984e 100644 --- a/cmd/codeaf/subharness.go +++ b/cmd/codeaf/subharness.go @@ -161,6 +161,50 @@ func gatheringGrant(turns, tokens int, fanIn store.DependencyFanIn) (int, int) { return turns + fanIn.Count, tokens + 2*landed } +// The re-dispatch grant: what a leaf that has been sent round again in place +// is given, in the one number the dispatch path states for ctxbudget. +const ( + // overrunRegrantNum over overrunRegrantDen is how much bigger one + // re-dispatch's token grant is than the grant of the attempt that ran + // out: three halves. A re-dispatch exists to finish a truncated tail, and + // half again is enough to finish one, where doubling would buy the whole + // run a second time. + overrunRegrantNum = 3 + overrunRegrantDen = 2 + + // overrunGrantCeiling is where the regrant stops adding: four flat leaf + // grants, which is room to run an ordinary leaf's whole text through twice + // over and therefore more than finishing a tail can ever cost. The bound + // is on what the ladder adds and not on what the leaf was granted — a + // fan-in that measured more than this keeps every token it measured, + // because the one thing a re-dispatch must never do is hand back room. + overrunGrantCeiling = 4 * chatLeafTokens +) + +// regrantAfterRunningOut is the token grant one claim of a leaf that has been +// re-dispatched in place is given, from the grant this claim measured and the +// attempt it is on. Attempt zero is the grant unchanged; every attempt after +// it grows by [overrunRegrantNum] over [overrunRegrantDen] for each re-dispatch +// before it, and the ladder adds nothing beyond [overrunGrantCeiling]. +// Turns are not regrown: a re-dispatch carries its banked turns as inputs +// rather than re-running them, so the turn ceiling was never what ran out. +// +// The growth is the point. A re-dispatch handed the room its predecessor ran +// out of re-runs the same brief to the same truncated ending — the meter stops +// it at the same place — so the runner's release (see resident's overrun +// settle) is worth a claim only if the room moves with it. The wall the leaf +// is given follows, because it is arithmetic over the same number. +func regrantAfterRunningOut(tokens, attempt int) int { + room := tokens + for ; attempt > 0 && room < overrunGrantCeiling; attempt-- { + room = room * overrunRegrantNum / overrunRegrantDen + if room > overrunGrantCeiling { + return overrunGrantCeiling + } + } + return room +} + // foldGrant sizes the whole run of a leaf that is going to make one model call. // // The gathering grant above is the right arithmetic for a node that has to go @@ -368,7 +412,13 @@ func installMeasuredRulers(settings config.Config, model string) *profile.Profil // it would hold the terminal behind a network round trip. Memoising it means the // identity seam and the surface that shows the model list are looking at the // same catalog rather than racing two fetches over one cache file. -var sharedCatalog = func() func(config.Config) *catalog.Catalog { +var sharedCatalog = newSharedCatalog() + +// newSharedCatalog builds the memoised accessor [sharedCatalog] is. It is a +// function rather than the value alone so a test can seat its own instance: +// the once is inside, and two runs in one process must not share a catalog +// pointed at whichever of them called first. +func newSharedCatalog() func(config.Config) *catalog.Catalog { var once sync.Once var resolved *catalog.Catalog return func(settings config.Config) *catalog.Catalog { @@ -376,10 +426,56 @@ var sharedCatalog = func() func(config.Config) *catalog.Catalog { resolved = catalog.LoadLazy(context.Background(), catalog.Options{ BaseURL: settings.BaseURL, APIKey: settings.APIKey, Dir: settings.ProfileDir, }) + // A tier row that says auto is answered from this catalog (config.AutoModels): + // the same non-blocking read, never a fetch, and set once at start-up so every + // headless door resolves the word against the list it already holds. + config.AutoModels = resolved.ModelsNow + // and the pool's index beside it, in the same one-time manner: a tier + // row that says auto is answered from the index this process was seated + // with, the seed when no cache is fresher, and the one fetch it makes + // runs in the background and never blocks this read. + wirePoolIndex(settings.ProfileDir) }) return resolved } -}() +} + +// autoSeatRowsBound is how long a headless door waits for the catalog's rows +// when the profile's pick or a tier row needs them. It is sized to cover the +// disk read of a cached catalog and nothing more. It is a variable because the +// test of the bound must not spend three seconds proving the bound is honoured. +var autoSeatRowsBound = 3 * time.Second + +// useAutoSeats seats this process's catalog under the seat ladder, and is what +// a headless door calls BEFORE it resolves its seats. +// +// The order is the whole of it. A tier row that says `auto` is answered from +// the rows already in hand ([config.AutoModels]), and every headless door +// climbed the ladder before it asked for a catalog at all — so the word read +// against nothing and landed on the family's table row on every run, on a +// machine whose catalog was sitting in its own cache file. It is the same lazy, +// memoised catalog every one of those doors goes on to use; asking for it a few +// lines earlier waits for nothing. +// +// AND WHEN THE ANSWER NEEDS THE ROWS, THE DOOR WAITS FOR THEM, within +// [autoSeatRowsBound]: a pick taken off the table ([config.CrewPickAt]) and a +// tier row that says auto ([config.AnyTierAutoAt]) are both computed from those +// rows, and the chat surface never met the defect because its picks happen after +// the warm has landed. The bound is a bound on the wait, not on the fetch — when +// it runs out the warm carries on in the background, the resolver falls to the +// family's table row exactly as it did before, and the seat's receipt names the +// rung that answered (`table`), so a run that fell says it fell. A profile with +// neither a pick nor an auto row reads no rows at all, and waits for nothing, +// the way it always has. +func useAutoSeats(settings config.Config) { + resolved := sharedCatalog(settings) + if config.CrewPickAt(settings.ProfileDir) == config.CrewPickTable && !config.AnyTierAutoAt(settings.ProfileDir) { + return + } + ctx, cancel := context.WithTimeout(context.Background(), autoSeatRowsBound) + defer cancel() + resolved.Warmed(ctx) +} // promisedWorker is the node's own answer to "who runs this", read in the order // admission settled it: the row's worker where there is one, the subtree's diff --git a/cmd/codeaf/subharness_run.go b/cmd/codeaf/subharness_run.go index acd4f568f1..201e0a49e6 100644 --- a/cmd/codeaf/subharness_run.go +++ b/cmd/codeaf/subharness_run.go @@ -110,6 +110,7 @@ func runSubharnessCommand(args []string) error { // One seat here — a saved program executes and never plans — climbed on the // same ladder every other headless door climbs, so a profile's crew reaches // this one too (config.ResolveSeats). + useAutoSeats(settings) seats := config.ResolveSeats(settings.ProfileDir, *model, "") applySeats(&settings, seats) fmt.Fprintln(os.Stderr, seats.Work.Report()) diff --git a/cmd/codeaf/usage.go b/cmd/codeaf/usage.go index d2a478a917..41587db657 100644 --- a/cmd/codeaf/usage.go +++ b/cmd/codeaf/usage.go @@ -472,7 +472,7 @@ func nearestCommand(typed string) string { // reason they are absent from the usage text: nothing types them. var knownCommands = []string{ "chat", "resume", "serve", "devices", "do", "plan", "revise", "run", "exec", - "show", "models", "notebook", "collections", "competence", "services", "wake", "doctor", + "show", "models", "pool", "notebook", "collections", "competence", "services", "wake", "doctor", "logs", "cache", "rebuild", "why", "manual", "version", "help", } diff --git a/docs/HEADLESS.md b/docs/HEADLESS.md index feb3cd749c..99a06632d7 100644 --- a/docs/HEADLESS.md +++ b/docs/HEADLESS.md @@ -153,7 +153,7 @@ kimi-k3:low` asked OpenRouter for a model id nobody publishes.) **Every run says which rung answered**, on stderr, before anything else: ``` -models: work deepseek/deepseek-v4-flash (crew frugal) · plan qwen/qwen3.8-27b (crew frugal) +models: work z-ai/glm-5.3-flash (crew frugal) · plan z-ai/glm-5.3-flash (crew frugal) models: work anthropic/whatever (--model) · plan follows the work model (default) ``` diff --git a/docs/changes/unreleased/1093-coverage-round-buys-once.md b/docs/changes/unreleased/1093-coverage-round-buys-once.md new file mode 100644 index 0000000000..887ad35e73 --- /dev/null +++ b/docs/changes/unreleased/1093-coverage-round-buys-once.md @@ -0,0 +1,16 @@ +--- +kind: changed +title: the coverage round buys once, and the envelope says when the requested work was done +pr: 1093 +surface: [resident, docs] +invalidates: + - "An unexercised-only finding bought a repair round at every later delivery gate, because the finding stands until a measurement closes it and nothing bounded how many rounds `no check exercises this behaviour` could raise. A job buys at most one such round now (`maxUnexercisedRounds`), and a second finding is recorded unclosed rather than funded." + - "Growth was bounded only by the round counter, the job-size ceiling, the wall and the daily dollar rail. A share-of-spend rail (`growthSpendShare`) refuses a round once growth has spent more than the requested work itself cost, counted from the gate that first found that work done, and journaled as `CauseSpendShare`." + - "`codeaf do --json` carried no figure for when the requested work was finished. `core_done_seconds` now appears, in seconds from the start, when a delivery gate first passed or found only a coverage gap, and is absent when no gate said so." +--- + +An unexercised-only finding is a measurement, so it stands until a measurement +closes it — right, but unbounded, it raised the same test-writing round at every +gate. A run whose named fix was committed minutes in spent the rest of its wall +and most of its bill writing more tests. The round is bounded to one, and growth +past the point the work was done is bounded against the bill that got it done. diff --git a/docs/changes/unreleased/1093-crew-default-family-is-all.md b/docs/changes/unreleased/1093-crew-default-family-is-all.md new file mode 100644 index 0000000000..92094fa422 --- /dev/null +++ b/docs/changes/unreleased/1093-crew-default-family-is-all.md @@ -0,0 +1,27 @@ +--- +kind: changed +title: The crew's default family is `all`, and both families' worker, careful and mastermind seats moved +pr: 1093 +surface: [chat, engine] +invalidates: + - "`models.crew.source` defaulted to `open`, so a profile that never answered the row resolved the open-weight table. The default is now `all`: `DefaultCrewSource = CrewSourceAll`, and a profile that never chose a family resolves the all family's balanced row — `google/gemini-2.5-flash`, `deepseek/deepseek-v4-flash-0731`, `z-ai/glm-5.3-flash`, `anthropic/claude-fable-5.1`, `anthropic/claude-fable-5.1`." + - "The five shipped tier defaults were the open table's balanced row. `DefaultReflexModel` is now `google/gemini-2.5-flash`, `DefaultHighModel` and `DefaultMastermindModel` are `anthropic/claude-fable-5.1`; `DefaultLowModel` and `DefaultWorkerModel` are unchanged. The identity the crew row rests on still holds: the five defaults are exactly the DEFAULT family's balanced row." + - "`config.CrewModels` and `config.CrewLine` answered the OPEN family. They answer the default family, which is now `all`; the open table is reached through `CrewModelsForSource(CrewSourceOpen, …)` and `CrewLineFor(CrewSourceOpen, …)`. A blank, misspelt or retired family word still folds to the default, which is `all` rather than `open`." + - "A profile that applied a preset under an older build now reads `custom`, because the worker, careful and mastermind seats moved in both families. The crew word, the `/crew` highlight and the seat rung all say `custom` until the preset is applied again, which writes the new five." + - "The open table's worker column is unchanged and its mastermind column is not: frugal now plans on `z-ai/glm-5.3-flash`, and balanced and max both plan on `z-ai/glm-5.3` rather than `moonshotai/kimi-k3`. The all table's worker holds `z-ai/glm-5.3-flash` through balanced and `openai/gpt-5.6-sol` and `anthropic/claude-opus-5` are in no preset of it any more." + - "The `/crew` listing named the family whenever the row was not `open`. It names it whenever the row is not the default family, so an `open` listing carries the `family · open models` line and an `all` one does not." +--- + +Both tables are read off one plot, computed seat by seat on 2026-09-16: the +expected bill a seat's own call shape runs up, built from the catalog's +published prompt, completion and cache-read prices, against that seat's quality +from its published intelligence, coding and agentic indexes. The worker and the +careful seats are priced as long cached loops and the mastermind as one-shot +calls, which is why the columns no longer climb together in either family. + +In the all family the worker stays on `glm-5.3-flash` through balanced because +the worker seat carries most of a task's tokens: a step there multiplies through +the whole bill, where a step on the careful or the mastermind seat is paid a +handful of times. The laws around the tables are untouched — the careful seat is +a different vendor from the worker in every preset of both families and still +sees images, and the reflex and small-work columns still never vary. diff --git a/docs/changes/unreleased/1093-crew-picked-from-row.md b/docs/changes/unreleased/1093-crew-picked-from-row.md new file mode 100644 index 0000000000..84d9026fbe --- /dev/null +++ b/docs/changes/unreleased/1093-crew-picked-from-row.md @@ -0,0 +1,43 @@ +--- +kind: added +title: The crew's seats are picked from a row — table, catalog or learn +pr: 1093 +surface: [chat, engine] +invalidates: + - "A seat on the computed rung read `crew frugal, computed`; it now reads `crew frugal, computed from the catalog`." + - "The crew word was derived from the five live seats; under a pick of catalog or learn it is derived from the five stored rows, because the live seats are computed ids the preset tables do not hold." + - "An unwritten seat read the preset's own table row on every surface; under a pick of catalog or learn the worker, careful work and mastermind seats are computed at the crew's preset instead, and a hand-typed model id still wins." +--- + +`models.crew.pick` is a new choice row beside the crew row: `table` (the default), +`catalog` and `learn`. The crew row keeps its meaning — how much to spend — and the pick +row says where the models for that budget come from when a tier row does not hold a model +id of its own. + +- `config.CrewPickAt` and `config.SetCrewPick` are the row's reader and writer; a word + the build does not know reads as the default and a writer refuses it, the way every + choice row folds. +- The ladder applies the pick in one seam both ladders call — `config.pickedSeat`, + beside `autoRow` in internal/config/auto.go. Under `catalog` or `learn` the worker, + careful work and mastermind seats are computed at the crew's preset + (`config.crewPresetUnder`), reflex and small work always read the table, a stored + model id that is not the preset's own table value wins with source `crew`, and a flag + or `CODEAF_MODEL`/`CODEAF_PLAN_MODEL` still outrank everything below them. +- `config.AutoPickWith(tier, family, preset, models, prior)` is the pick with the + measured quality named: `catalog` passes a nil prior, `learn` passes the Model Pool's + role-quality prior (`config.autoPrior`), and `config.AutoPick` keeps its behaviour — + prior included — for the bare `auto` word a tier row may hold. +- `config.SeatLearned` is the new rung. `Seat.Rung()` says `crew balanced, computed from + the catalog` for `SeatComputed` and `crew balanced, learned` for `SeatLearned`; + `Seats.Report()`, `Seats.Line()` and the receipts follow. The headless doors read the + same pick through `config.ResolveSeats`, so a run from the shell is seated exactly as + the conversation is. +- internal/tui3: the row is on the Providers tab directly under the crew word + (`picked from`, a cycle widget), `/crew` takes the three words beside the presets and + refuses an unknown word by naming all six, and the crew word and status segment read + `balanced · learn` / `crew balanced · learn` when the pick is off the table. The + chooser names the pick on a reading line under the presets, and a crew applied under a + pick confirms with the pick named beside the preset (`config.CrewSummaryPick`). +- internal/manual: the class-row section is now `Where the seats are picked from — + table, catalog, learn`, with the bare `auto` word kept inside it as the per-seat + alias, and the headless ladder page names the two rung words. diff --git a/docs/changes/unreleased/1093-crew-picking-and-pool-reading-packages.md b/docs/changes/unreleased/1093-crew-picking-and-pool-reading-packages.md new file mode 100644 index 0000000000..e665704bc9 --- /dev/null +++ b/docs/changes/unreleased/1093-crew-picking-and-pool-reading-packages.md @@ -0,0 +1,27 @@ +--- +kind: added +title: Three pure packages — a crew picker, an additive tally and a measurement-index reader +pr: 1093 +surface: [engine] +invalidates: [] +--- + +Standard library only, no disk, no network, and nothing on a surface wired to +them yet. + +- `internal/crewpick` scores every (worker, high, mastermind) combination a + candidate list can field by the bill its seat volumes run up against the mean + of its seat qualities, and reads the frugal, balanced and max picks — and a + knob between them — off the non-dominated front. A candidate that publishes + some but not all of its three indexes is scored on the ones it publishes, + each missing one estimated from the call's own candidates carrying the pair + and never above the largest measured value of that index; `Crew.Estimated` + says which of a pick's indexes were estimated, and a tie in bill and quality + runs to the crew on fewer of them. Only a candidate publishing no index at + all is out of the running. +- `internal/pool/tally` keeps per-address sufficient statistics (n, sum, sum of + squares) and paired-comparison counts, so two sheets merge by addition in any + order and marshal to a byte-identical schema-1 document. +- `internal/pool/index` parses a schema-versioned JSON document of per-model + measurements into an immutable value every goroutine can share, resolving + model aliases and ignoring fields, metric kinds and dims it does not know. diff --git a/docs/changes/unreleased/1093-crew-tables-move-to-the-measured-seats.md b/docs/changes/unreleased/1093-crew-tables-move-to-the-measured-seats.md new file mode 100644 index 0000000000..cadb6817ca --- /dev/null +++ b/docs/changes/unreleased/1093-crew-tables-move-to-the-measured-seats.md @@ -0,0 +1,15 @@ +--- +kind: changed +title: Six cells of the fixed crew tables moved to the measured seats +pr: 1093 +surface: [chat, engine] +invalidates: + - "The all family's frugal careful seat is `qwen/qwen3.8-max-0902` (was `google/gemini-3.8-flash`). Its balanced mastermind is `anthropic/claude-opus-5` (was `anthropic/claude-fable-5.1`). Its max row works on `z-ai/glm-5.3` (was `anthropic/claude-fable-5.1`), checks on `anthropic/claude-fable-5.1` (was `openai/gpt-6-astra`) and plans on `anthropic/claude-opus-5` (was `anthropic/claude-fable-5.1`). The open family's frugal row works on `z-ai/glm-5.3-flash` (was `deepseek/deepseek-v4-flash-0731`)." + - "`config.DefaultMastermindModel` is `anthropic/claude-opus-5`, so the five shipped defaults are still exactly the default family's balanced row. An untouched profile plans on opus-5 and reads the crew as `balanced`; a profile that pinned the mastermind on the old default now reads `custom` until it pins again." + - "Max now differs from balanced only in the worker seat, so an `auto` row on the worker can no longer tell the two presets apart — the stored-rows reading answers `balanced`, the default preset, and the worker's own id is what identifies max to the seam. The open family's frugal row has worker and careful on the same vendor, `glm-5.3-flash`, the one standing exception to the second-vendor law." +--- + +The cells move to the same measured plot the tables already name: expected bill +under each seat's call shape against published quality, nothing quoted. Max +spends on the seat that pays most of a task's bill; the careful and mastermind +seats settle at balanced and hold through max. diff --git a/docs/changes/unreleased/1093-fuel-asks-an-installed-tariff-first.md b/docs/changes/unreleased/1093-fuel-asks-an-installed-tariff-first.md new file mode 100644 index 0000000000..bd93fad0d5 --- /dev/null +++ b/docs/changes/unreleased/1093-fuel-asks-an-installed-tariff-first.md @@ -0,0 +1,15 @@ +--- +kind: changed +title: The fuel meter asks an installed tariff first, and its table holds the ids this build ships +pr: 1093 +surface: [engine] +invalidates: + - "`orchestrate.PriceOf` answered from one package-level table and nothing else. An installed `PriceSource` is asked first and the table is the fallback behind it; `UsePrices` installs or removes one, and `CatalogPrices` adapts a per-token catalog reader into the per-million shape the meter uses." + - "The fuel table now holds exactly the nine ids the shipped defaults and crew tables name: `google/gemini-2.5-flash`, `mistralai/mistral-nemo`, `deepseek/deepseek-v4-flash-0731`, `z-ai/glm-5.3-flash`, `z-ai/glm-5.3`, `moonshotai/kimi-k3`, `qwen/qwen3.8-max-0902`, `anthropic/claude-fable-5.1` and `anthropic/claude-opus-5`. `anthropic/claude-opus` and `openai/gpt-5` are gone, and both now meter at the unpriced rate rather than at a row of their own." + - "An orchestrated run metered against the fuel table whenever a call reported no cost of its own. It meters against the catalog's published price when the session has one, and falls back to the table only for a model the catalog publishes nothing for." +--- + +The seam is a function value, so the meter never imports a catalog: the one +place an orchestrated run is built already holds a reader and installs it +there. A session with no reader installs nothing and leaves whatever is +installed alone, because the seam is shared by every run in the process. diff --git a/docs/changes/unreleased/1093-headless-doors-wait-for-the-rows.md b/docs/changes/unreleased/1093-headless-doors-wait-for-the-rows.md new file mode 100644 index 0000000000..837ff89ab0 --- /dev/null +++ b/docs/changes/unreleased/1093-headless-doors-wait-for-the-rows.md @@ -0,0 +1,20 @@ +--- +kind: fixed +title: the headless doors wait for the catalog rows a pick reads +pr: 1093 +surface: [engine] +invalidates: + - "`codeaf do`, `codeaf exec` and every door through `useAutoSeats` resolved their seats while the lazy catalog was still warming, so a pick taken off the table — or a tier row that says `auto` — read no rows and fell to the family's table row, reported as `crew balanced, table`, on a machine whose catalog was already cached beside the profile. They now wait for the warm within a three-second bound when the profile needs the rows, and fall exactly as before when the bound runs out — with the rung word saying which happened." + - "internal/catalog's `Warmed(ctx)` is the bounded wait at the warm's door — true at once for an eagerly loaded catalog, false at the bound for one still fetching — and `config.AnyTierAutoAt` answers whether any tier row says `auto`, the second reason a door waits, beside `config.CrewPickAt`." +--- + +A pick off the table and a tier row that says `auto` are both computed from the +rows the process already holds (`config.AutoModels`), which the headless doors +set from a lazy catalog's non-blocking read. The chat surface never met the +defect because its picks happen after the warm has landed; a headless door's +picks happen a line after the read, so the warm was always still in flight and +the resolver saw no rows. The wait is bounded — three seconds, sized to the +disk read of a cached catalog — and the fetch is never waited on past it: a +cold cache on a slow network still resolves from the family's table row, and +the seat's receipt still names the rung that answered, so a run that fell says +it fell. A profile with neither a pick nor an auto row waits for nothing. diff --git a/docs/changes/unreleased/1093-no-git-command-without-a-directory.md b/docs/changes/unreleased/1093-no-git-command-without-a-directory.md new file mode 100644 index 0000000000..277f823017 --- /dev/null +++ b/docs/changes/unreleased/1093-no-git-command-without-a-directory.md @@ -0,0 +1,14 @@ +--- +kind: fixed +title: No git command runs without a directory, and the checkout guard reads a linked worktree's own share +pr: 1093 +surface: [engine] +invalidates: + - "`worktreeDirtIn` and `UnsavedEditsNote` built `git -C ` themselves, and `git -C \"\"` is a no-op, so an empty directory ran the command where the process stood. Both go through `gitWith` now (`gitContext` is the same call under a caller's context), where the refusal of an empty directory and the pinned environment live once." + - "`taskTree.releaseLanded` removed a worktree and deleted its branch with no empty-root guard and without the root lock. It refuses a tree with no root or directory and takes the root lock, as `releaseKept` does." + - "The session package's checkout guard read branches and worktree registrations for the whole shared clone, so from a linked worktree a sibling's task/* churn was reported as the run's damage, and its live-codeaf probe walked `/proc`, which does not exist on darwin. From a linked worktree the branch reading is skipped and the worktree reading keeps only registrations under the checkout; the probe reads the process table." +--- + +A command with no directory runs wherever the process happens to be. Every +spelling of a git call in this package is a wrapper of one function that +refuses that, and the two that had gone round it are wrappers of it now. diff --git a/docs/changes/unreleased/1093-pool-client.md b/docs/changes/unreleased/1093-pool-client.md new file mode 100644 index 0000000000..ce92c5422e --- /dev/null +++ b/docs/changes/unreleased/1093-pool-client.md @@ -0,0 +1,29 @@ +--- +kind: added +title: The pool client gains the relay's real addresses, an install nonce, and the index signer's key +pr: 1093 +surface: [engine] +invalidates: + - "The pool's default addresses were `https://pool.invalid/index.json` and `https://pool.invalid/submit`. They derive from one base now — `DefaultRelayURL` is `https://codeaf.agentfield.ai/pool`, the index at `…/index.json`, the submit at `…/v1/rows` — and `CODEAF_MODEL_POOL_RELAY_URL` moves the base. The two per-address pins (`CODEAF_MODEL_POOL_URL`, `CODEAF_MODEL_POOL_SUBMIT_URL`) still override, and `pool show` prints a `relay` line beside the two derived addresses." + - "`pool verify` refused with `no public key built into this build; pass --key` because the build carried no key. The index signer's key is decoded into `poolPublicKeys` at init now, so `verify` fetches without a flag; `--key` checks a document signed under some other key, and the new `models.pool.public_key` row (env `CODEAF_MODEL_POOL_PUBLIC_KEY`) replaces the built-in key when set — and only it: a stored word that does not decode trusts nothing." + - "A stored pool index was read only at the fetch that brought it. `refreshPoolIndex` now runs daily under the config's TTL from a build that carries the key, and a changed document is still read at the next start." + - "An outbox batch carried no identity. Every batch over http now rides `X-Codeaf-Install` with the install's nonce — 32 lowercase hex in `/install`, replaced when it stops being that shape — and when `model_pool` is on the outbox is pushed after each judged run and once at start-up, budget 5 s each, errors at debug level only." +--- + +The relay wants a nonce on every batch and signs its index under one key; this +wave gives the client both, so an install with `model_pool` on reads the +signed index and sends its rows the day the relay answers. Until it answers, +every path falls back the way it did: the cache, the seed, and the install's +own sheet, with nothing said at a person. +The relay's index has a second address — the same signed document copied to +GitHub — held as `DefaultMirrorURL` (env `CODEAF_MODEL_POOL_MIRROR_URL`, a +value set and empty to turn the fallback off) and printed by `pool show` beside +the other addresses. A refresh that fails for any reason other than a signature +failure falls through to the mirror, which shares the cache directory, so the +document with the higher version is the one kept; a signature failure is a +statement about the primary's bytes and does not fall through at all. `pool +status` now asks the relay, and the mirror when the relay does not answer, under +a three-second budget and TTL 0, and prints one line: whether each answered, +the index version it served, or the reason it did not — with `--key` to check a +document signed under a key other than the one built in. Status still exits 0 +whether or not anything answered. diff --git a/docs/changes/unreleased/1093-pool-delivery-settings-and-fetch-packages.md b/docs/changes/unreleased/1093-pool-delivery-settings-and-fetch-packages.md new file mode 100644 index 0000000000..da13d0e497 --- /dev/null +++ b/docs/changes/unreleased/1093-pool-delivery-settings-and-fetch-packages.md @@ -0,0 +1,29 @@ +--- +kind: added +title: Three more `internal/pool` packages — an outbox, a settings resolver and a signed-document fetcher +pr: 1093 +surface: [engine] +invalidates: [] +--- + +Standard library only, and nothing on a surface wired to them yet. + +- `internal/pool/outbox` keeps small measurement rows in a local + newline-delimited JSON file and hands them to a destination in batches, over + http or into a file. A row that arrived is marked sent and the mark outlives + the handle, so nothing is delivered twice; a batch that did not arrive leaves + its rows and every row after it pending, with an error naming what failed. + The file is capped from the old end, the payload is stored compacted, and a + caller never has a full disk or a dead destination raised at it as anything + but a returned error. +- `internal/pool/poolcfg` resolves how the pool behaves from one stored setting + and an environment handed in as a function, so there is no clock, no disk and + no reading of the process environment in it. Every field carries where its + value came from — `default`, `setting`, `env` or `ci` — and the addresses it + accepts are https, http on loopback only, `file://` and absolute paths. +- `internal/pool/pull` fetches a JSON document, checks a detached ed25519 + signature at the same location with `.sig` appended, and keeps the last good + copy under a cache directory so a run that cannot reach the source still has + an answer. A young cache skips the network entirely, an ETag turns into a + conditional request, a version lower than the cached one is refused, and every + failure hands back the cached copy beside a non-nil error. diff --git a/docs/changes/unreleased/1093-pool-judge-hook.md b/docs/changes/unreleased/1093-pool-judge-hook.md new file mode 100644 index 0000000000..2a209244e1 --- /dev/null +++ b/docs/changes/unreleased/1093-pool-judge-hook.md @@ -0,0 +1,19 @@ +--- +kind: added +title: a landed task is scored by a model outside the crew into the own sheet, outbox and judge seat +pr: 1093 +surface: [chat, engine] +invalidates: + - "The Model Pool's own sheet (`own.json` under the pool directory) could only hold scores a machine put there by hand. After a task lands in a chat-door conversation, a judge model outside the crew now scores each seat the work ran on, the scores are observed into that sheet, and the picker reads the new cells in the same process at once (`config.AutoOwnCells` repointed after every landing)." + - "The outbox (`outbox.jsonl` beside the sheet) gained the same rows when `model_pool` is `on` and a submit address is set; `read` keeps them local and `off` asks no judge at all." + - "The judge's provider calls are billed to the `judge` seat in the usage ledger, with the dollars from the model's published price when the provider sent no receipt of its own." +--- + +The engine grew the one seam it was missing: `session.Config.TaskLanded`, called +once per landed node on its own goroutine through `guard.Go`, carrying the +node's record as the landing left it plus the worker's model and the model its +checking pass ran on (`session.TaskLanding`). The chat door wires it to the +pool's judge (`cmd/codeaf/poolrecord.go`): pick a judge outside the crew +(`judge.Pick`), ask one question per held seat (`judge.Judge`), record what came +back (`record.Recorder`), and hand the new cells to the picker. A mode that +forbids reading builds no hook, and a nil hook costs the engine nothing. diff --git a/docs/changes/unreleased/1093-pool-judge.md b/docs/changes/unreleased/1093-pool-judge.md new file mode 100644 index 0000000000..c9efbea63c --- /dev/null +++ b/docs/changes/unreleased/1093-pool-judge.md @@ -0,0 +1,24 @@ +--- +kind: added +title: A pure package that scores a finished run seat by seat, by a judge outside the crew +pr: 1093 +surface: [engine] +invalidates: [] +--- + +Standard library and `internal/catalog` only, no disk, no network, and nothing +on a surface wired to it yet. + +- `internal/pool/judge` reads a run's record — its brief and deliverable, its + report, claim and ending, the files it wrote and the checks it ran, and which + model held each of the worker, high and mastermind seats — and asks one + question per seat, in role order, through a caller-supplied `Ask`. Each answer + is read as one JSON object carrying a 0-100 score and a one-sentence reason, + on the same scale crewpick reads seat quality on and a pool records its + `role_quality` metric in. A seat whose model is the judge's own is skipped; a + seat whose answer cannot be read fails alone, and the scores obtained come + back beside an error naming it. The caller owns the model, the transport and + the bill, and bills the judge's calls to its own seat. +- `judge.Pick` chooses that judge from a catalog: the cheapest row whose + published coding index reaches `judge.DefaultFloor`, that carries tools, that + no crew seat holds and whose vendor is not the worker's, ties broken by id. diff --git a/docs/changes/unreleased/1093-pool-recorder.md b/docs/changes/unreleased/1093-pool-recorder.md new file mode 100644 index 0000000000..db5612ea7e --- /dev/null +++ b/docs/changes/unreleased/1093-pool-recorder.md @@ -0,0 +1,37 @@ +--- +kind: added +title: A pure package that records a run's judged scores into an own sheet and outbox rows +pr: 1093 +surface: [engine] +invalidates: [] +--- + +`internal/pool/record` turns the scores `judge.Judge` answered into the two +records an install keeps of them. `Recorder.Record` observes every valid score +into the sheet it holds, under the `role_quality` metric the crew picker reads +seat quality from, and appends one row per score to its outbox when it holds +one; a nil outbox records locally only. A score whose role is not one of the +judged seats, or that is not on the 0-100 scale, is skipped and named in the +error beside the others that were recorded, and the sheet is observed whatever +the outbox does. + +- `SaveSheet` and `LoadSheet` keep that sheet at `own.json` under the pool + directory — a missing file is an empty sheet, a save is a temp-file rename + into a 0600 file beside a 0700 directory — and `record.Cells` reads the + sheet's plain `role_quality` cells back as the cells a prior reads, sorted + by seat then model. +- `config.AutoOwnCells` is the start-up seam beside `config.AutoIndex`, and + `autoPrior` folds the own cells into the index's prior seat by seat through + `crewpick.MergePriors`, the means combined by observation count, at a floor + of one rather than the index's min_installs: an install's own scores are its + own evidence. A nil seam changes nothing. +- `cmd/codeaf` reads the own sheet once at start-up beside the index + (`wirePoolIndex`); a sheet that does not parse is said under the debug + record's switch and read as absent, not as a fault a pick stops for. + `codeaf pool show` and `pool status` say what it holds — `own sheet: + cells, observations`, or `own sheet: none` — with the same counts under + `--json`'s `own`. +- The seat vocabulary a usage row's `seat` field carries grows a seventh + word, `judge`, between mastermind and talk: the seat a run's judge is + billed to, mapped from no tier and no role. No tier and no registered role + answer it; the word reaches a row through the billing door's seat argument. diff --git a/docs/changes/unreleased/1093-pool-relay.md b/docs/changes/unreleased/1093-pool-relay.md new file mode 100644 index 0000000000..0f5350fcb2 --- /dev/null +++ b/docs/changes/unreleased/1093-pool-relay.md @@ -0,0 +1,15 @@ +--- +kind: added +title: A Cloudflare Worker relay that folds installs' rows into a signed, versioned pool index +pr: 1093 +surface: [engine] +invalidates: [] +--- + +The relay under `relay/` accepts NDJSON measurement rows at `POST /pool/v1/rows`, +keyed by the install's `X-Codeaf-Install` header, folds each install's rows into +per-install per-day running totals in KV, and once an hour publishes a signed, +versioned JSON index at `GET /pool/index.json` beside its Ed25519 detached +signature at `GET /pool/index.json.sig`. The document is what +`internal/pool/index` reads and is signed with a key whose public half is built +into the client. diff --git a/docs/changes/unreleased/1093-pool-seed-index.md b/docs/changes/unreleased/1093-pool-seed-index.md new file mode 100644 index 0000000000..bc17fb9839 --- /dev/null +++ b/docs/changes/unreleased/1093-pool-seed-index.md @@ -0,0 +1,25 @@ +--- +kind: added +title: The Model Pool's index is seated at start-up from a seed in the binary +pr: 1093 +surface: [chat, engine] +invalidates: + - "`config.AutoIndex` was a seam nothing in the binary set, so a tier row that says `auto` picked on the catalog's published figures alone. `cmd/codeaf` seats it at start-up from the seed the binary carries or a fresher cached document, so the pool's measured quality reaches the picker." + - "The binary carried no index of its own, and `codeaf pool show` said only `no index cached yet`. `internal/pool/index` embeds a seed document, and show names it — `no index cached yet · built-in seed of `." +--- + +`internal/pool/index` embeds a seed measurement document (`Seed`, `SeedIndex`) +so a machine that has never fetched an index still has numbers to pick a seat +against. `cmd/codeaf` reads the cached `doc.json` beside that seed through +`index.Fallback` once at start-up — the newer `generated` day wins — and hands +the result to `config.AutoIndex` where `config.AutoModels` is already seated, on +every door that resolves a seat (`sharedCatalog`, and the three chat doors). +The read is start-up work and never on a run's path. + +A background goroutine, started where the index is seated, refreshes the cache +through `pull` only when the mode allows reading and the build carries a public +key; with no key compiled in today it starts nothing. A fresh document lands in +the puller's own cache and is read at the next start — the running process keeps +the index it was seated with. `codeaf pool show` names the seed when no cache is +present, and its `--json` shape carries the index it read with a `source` field +naming `cache` or `seed`. diff --git a/docs/changes/unreleased/1093-redispatch-room-and-refusal.md b/docs/changes/unreleased/1093-redispatch-room-and-refusal.md new file mode 100644 index 0000000000..542ac6dd80 --- /dev/null +++ b/docs/changes/unreleased/1093-redispatch-room-and-refusal.md @@ -0,0 +1,15 @@ +--- +kind: changed +title: A re-dispatch after running out is granted more room, and one that banks nothing new is refused +pr: 1093 +surface: [resident, docs] +invalidates: + - "A leaf that ran out of its token budget was re-dispatched with the identical budget and the identical brief, up to the round cap, and bought the same truncated ending each time. The dispatch's grant is attempt-aware now: `regrantAfterRunningOut` in `cmd/codeaf/subharness.go` grows the token grant by three halves per re-dispatch, never above `overrunGrantCeiling` (four flat leaf grants; a fan-in that measured more keeps every token it measured), and the wall the leaf is given follows the larger grant." + - "A leaf that ran out was requeued whenever anything at all was banked, for as many rounds as the cap allowed. A re-dispatch that banks nothing the attempt before it had not banked is failed now, with the refusal named in the node's error; the count it is measured against is the one the hand-on release journaled, read back through `store.ReleasedTurnsFor`." + - "The `--json` envelope had no field for in-place re-dispatches. `redispatches` counts them across the run's nodes off the journal (`store.NodeRedispatches`), and is absent rather than zero when there were none." +--- + +The two halves are one mechanism: the grant moves with the attempt, and the +record says whether moving it moved anything. A re-dispatch is worth buying only +when the room grows and the work follows; either half alone is the old loop with +a bigger bill. diff --git a/docs/changes/unreleased/1093-tier-row-auto-computes-from-catalog.md b/docs/changes/unreleased/1093-tier-row-auto-computes-from-catalog.md new file mode 100644 index 0000000000..0d3ae8a28f --- /dev/null +++ b/docs/changes/unreleased/1093-tier-row-auto-computes-from-catalog.md @@ -0,0 +1,36 @@ +--- +kind: added +title: A tier row that says `auto` computes its model from the catalog +pr: 1093 +surface: [chat, engine] +invalidates: [] +--- + +Any of the five tier rows (`models.tiers.*`) may hold the bare word `auto`, +case folded: the seat's model is computed from the catalog's own published +figures through `internal/crewpick` — the three indexes against the three +prices, under each seat's call shape — every time the row is read. The word +stays on disk; the id is derived on every read. + +- `config.AutoValue`, `config.IsAuto` and `config.AutoPick` are the word and + its pure pick; `config.AutoModels` is how the catalog reaches seat + resolution, set once at start-up from the binary's non-blocking read, + never a fetch. Nil is an ordinary state, not an error. +- Two new rungs on the seat ladder: `config.SeatComputed` and + `config.SeatTable`. An auto row resolves on both ladders + (`config.TierSeatAt` and `config.ResolveSeats`) through one seam — to the + computed id, or, when nothing can be computed, to the family's table row + for the preset the stored rows name. It never resolves to `auto` and never + to empty. +- A seat on either new rung names the preset it ran at: `crew frugal, + computed` on a run's receipt. +- The preset an auto row runs at is read from the profile's other stored + rows, an auto row matching whichever preset is being compared. **When the + stored rows match more than one preset, the default preset — `balanced` — + wins the tie**, which is also what a profile matching none of them reads. + Since `max` differs from `balanced` only in the worker seat, a crew with + `auto` on the worker and the shipped rows elsewhere matches both and reads + as `balanced`; the worker's own id is what identifies `max` to the seam. +- Rows that don't say auto are unchanged, and so is everything above the + crew: a flag, `--plan-model` and the environment still outrank an auto + row, and a flag whose text is `auto` is handed on whole. diff --git a/docs/changes/unreleased/1093-usage-rows-name-their-seat.md b/docs/changes/unreleased/1093-usage-rows-name-their-seat.md new file mode 100644 index 0000000000..10847f2cc2 --- /dev/null +++ b/docs/changes/unreleased/1093-usage-rows-name-their-seat.md @@ -0,0 +1,24 @@ +--- +kind: changed +title: Every usage row names the seat the call ran under +pr: 1093 +surface: [engine] +invalidates: + - "The usage ledger's header said a row could not answer which class of model took the money, and `UsageLine.Role` was carried by only a handful of auxiliary calls. A row now carries `seat` — one word of a closed vocabulary, `reflex`, `low`, `worker`, `high`, `mastermind` or `talk` — and `role` is written by every call that goes through the role registry, so a spend page can say whether the money went on the seat that does the work or the seat that thinks." + - "`session.UsageLine` gained a `Seat` field. It is `omitempty` like every additive field before it, so every row already in the file still decodes and reads back with an empty seat and an empty role, which is the truth about it." + - "`Agent.recordUsageLine` took five arguments — the usage, the model, the role, the lane facts and the reconciled flag. It takes the one `bankedCall` those came off, because the seat needs a sixth fact from the same value and a door with six arguments is a door somebody passes in the wrong order." +--- + +The seat is derived and never typed at a call site: an agent's own turns bill +to the seat its kind answers for — `talk` for a conversation, `worker` for a +task node, `high` for the checker and a repair round — and every other row +falls back to the tier the role registry already has for that role +(`session.SeatOfRole` is `roles.TierOf` followed by one hand-written table of +five tiers to five words). A call that resolved its model outside the registry +— a media pin, a document reader, a tool ask — is nobody's seat and stays +wordless rather than guessed, and no free text ever reaches the field. + +The seat names the chair, not the id that answered, so it stays true of a row +whose model was a fallback, a pin or a rescue. It is NOT the router's slot: +nothing in the program records which slot a call ran under, and a page that +labelled this column with those words would be inventing the join. diff --git a/docs/changes/unreleased/1101-crew-pool-quality-prior.md b/docs/changes/unreleased/1101-crew-pool-quality-prior.md new file mode 100644 index 0000000000..1a6b35df1e --- /dev/null +++ b/docs/changes/unreleased/1101-crew-pool-quality-prior.md @@ -0,0 +1,15 @@ +--- +kind: changed +title: The crew picker blends a measured pool rating into a seat's quality by its observation count +pr: 1101 +surface: [engine] +invalidates: + - "`crewpick.SeatQuality` and `crewpick.Front` read a seat's quality from the catalog's published indexes alone. They now have `SeatQualityWith` and `FrontWith`, which blend a `crewpick.Prior` — a pool's measured quality per seat, per canonical model id — into each seat by N/(N+PriorWeightAt): a rating the prior holds with a positive count moves the seat towards the rating's mean, and `Crew.Measured` says which seats a rating entered. `SeatQuality` and `Front` keep their signatures and read no prior." + - "`config.AutoPick` resolved a seat from the catalog's published figures and nothing else. It now reads `config.AutoIndex`, a Model Pool index seam beside `config.AutoModels`, and blends the index's `role_quality` cells (a gaussian metric only, through `config.PoolQualityMetric`) into the pick through `crewpick.FrontWith`. A nil index leaves the answer exactly what it was." +--- + +The table of published indexes is a statement about models, and a pool that has +actually run a model on a seat holds something the table cannot: what it scored +there, over however many observations. The blend carries each by its evidence, +so a handful of pool ratings nudges a seat and a large pool of them decides it, +while a picker with no index in hand is unchanged. diff --git a/docs/changes/unreleased/1102-pool-setting-and-verb.md b/docs/changes/unreleased/1102-pool-setting-and-verb.md new file mode 100644 index 0000000000..05e75ad959 --- /dev/null +++ b/docs/changes/unreleased/1102-pool-setting-and-verb.md @@ -0,0 +1,23 @@ +--- +kind: added +title: The Model Pool reaches its first surface — a `model_pool` setting and a `codeaf pool` verb +pr: 1102 +surface: [chat, engine] +invalidates: + - "`internal/pool` was reachable from nothing outside itself. `config.ModelPoolAt` resolves the stored word and the four pool names, and `codeaf pool show|status|verify` answers from it, so the packages have a surface and the resolver in internal/config is the one place the process environment is read for the pool." + - "`CODEAF_MODEL_POOL` sat in `OperatorEnvPins` with a comment saying the word had no row because nothing in the binary called poolcfg. It fronts the `model_pool` row now — choices `on`/`read`/`off`, default `on` — and renders through the row the way `CODEAF_DOC_ENGINE` does; the two URL names and the TTL stay plumbing." + - "`codeaf --help` was at its 110-line cap. The pool entry takes two lines, paid for by folding the devices-revoke sentence and dropping `--model` from `plan new`'s synopsis, both carried by the per-command pages; exec keeps `--out` and `--debug`." +--- + +The setting is one choice row in the models group, pinned by +`CODEAF_MODEL_POOL`. The verb has three forms: `show` — also bare +`codeaf pool` — prints the resolved config with the word saying where each +value came from (`default`, `setting`, `env` or `ci`) and the cached index +with its age, or `no index cached yet`; `status` adds the outbox's pending +count and whether the mode allows sending and reading; `verify` fetches a +fresh index through `pull` with TTL 0 and checks its detached ed25519 +signature under `--key` or a built-in key, refusing at the door with exit 2 +while no build carries the published key and leaving on exit 1 when a fetch +or a signature fails. `--json` prints one object on every form that answers. +Nothing sends anything yet: no run records a measurement, so the outbox is +always empty and `verify` is the one form that reaches the network. diff --git a/docs/design/model-pool/Makefile b/docs/design/model-pool/Makefile new file mode 100644 index 0000000000..08d0fd4ade --- /dev/null +++ b/docs/design/model-pool/Makefile @@ -0,0 +1,27 @@ +# Builds pareto-crewing.pdf: the figures from data/*.csv, then the paper. +# make stale figures + pdf +# make figures redraw every figure and table +# make clean remove build products but keep the pdf + +FIGURES := fig-front.pdf fig-cells.pdf fig-shrink.pdf fig-regret.pdf fig-semibandit.pdf tab-regret.tex tab-semibandit.tex + +all: pareto-crewing.pdf + +figures: + python3 figures.py + +# One run of the script writes every figure and table; the stamp is what make +# dates them by, so old makes without grouped targets work too. +$(FIGURES): figures.stamp +figures.stamp: figures.py data/measured-crews.csv data/seed-cells.csv + python3 figures.py + touch $@ + +pareto-crewing.pdf: pareto-crewing.tex $(FIGURES) + latexmk -pdf -interaction=nonstopmode -halt-on-error pareto-crewing.tex + +clean: + latexmk -c pareto-crewing.tex + rm -f pareto-crewing.bbl figures.stamp + +.PHONY: all figures clean diff --git a/docs/design/model-pool/REVIEW-2.md b/docs/design/model-pool/REVIEW-2.md new file mode 100644 index 0000000000..d18220d551 --- /dev/null +++ b/docs/design/model-pool/REVIEW-2.md @@ -0,0 +1,60 @@ +# Second review of docs/design/model-pool/pareto-crewing.tex + +Referee report on the revision; the first is `REVIEW.md` (written against the draft then called `learn.tex`), the authors' answer `REVIEW-RESPONSE.md`. Every check below was made against this branch's code and data: `internal/pool/{index,judge,record,tally,outbox,pull,poolcfg}`, `internal/crewpick`, `cmd/codeaf/pool*.go`, `internal/config/{settings,auto}.go`, `relay/src/*.js`, `relay/wrangler.toml`, `.github/workflows/pool-mirror.yml`, `docs/design/model-pool/{figures.py,RUNBOOK.md,data/*.csv}`, `README.md` and the settings hint in `internal/tui3/settings.go`. Nothing was executed; no test suite, no `figures.py`. + +## Verdict + +**Minor revision.** The central mathematics is now right: Proposition 4's capped posterior derives correctly from the stated hierarchy including the bias term (checked end to end below), the drift moment no longer halves the estimate, and Proposition 3's statement is repaired. Every number quoted in Section 8 matches the committed tables exactly, the experiments run on all 23 chat-door crews, the row schema matches `record.Row` and the relay's validator field for field, and the Status paragraph is for the first time an accurate map of the branch — every claim in it checks against the tree except one word. What remains is a set of sentences that describe the relay as more merged and more complete than the code under `relay/` is (the join on the write path, drift and the constants in the hourly job, pool status's relay line, the read-side installs floor), one false sentence left inside a corrected proof, and one provenance number (72 runs over nine tasks) with no source in the repository. Each is a sentence or a small patch; none touches the paper's results. Minor revision. + +**Checks that passed.** (i) *Proposition 4* (:331–338). Under hierarchy (6), pool rows carry marginal variance $\sigma^2+\tau_1^2$ about $\mu_{rm}$, so their effective count at variance $\sigma^2$ is $\tilde n=n^{\mathrm{pool}}\sigma^2/(\sigma^2+\tau_1^2)=n^{\mathrm{pool}}\ell/(\ell+1)$; the posterior of $\mu_{rm}$ has precision $K/\sigma^2$, $K=k+\tilde n$, mean the $k{:}\tilde n$ weighted average of $\hat q_r(m)+b$ and $\bar y^{\mathrm{pool}}$; integrating $\mu_{rm}$ out gives $\mu^u$ an external precision $1/(\sigma^2/K+\tau_1^2)=W/\sigma^2$ with $W=K\ell/(K+\ell)\le\ell$, and combining with the own rows yields exactly (15)–(16) with $k^{\ast}+n^{\ast}=W$ in the ratio $k{:}\tilde n$. The Corollary is the $\ell=\infty$ limit ($W\to k+n^{\mathrm{pool}}$, $b=0$) and matches `crewpick.blendQuality` with `PriorWeightAt = 30` (crewpick.go:479) and the fold in internal/config/auto.go:169–176. (ii) *Drift* (:264): adjacent-day means have variance $\omega^2+\sigma^2(1/n_d+1/n_{d-1})$; the bracket subtracts exactly that, so (13) now estimates $\omega^2$, and the steady-state relation is untouched. (iii) *Section 8 against the tables*: Thompson 0.64 vs table 0.97 at $N=400$ (gap \$0.33, a third of 0.97), $\varepsilon$-greedy 0.76 (\$0.12 behind), crew-mean-score 0.31, crew-clean-bit 0.54, role 0.15 (half of 0.31, a third of 0.54) — all match `tab-regret.tex` and `tab-semibandit.tex` exactly; 23 chat-door crews (data/measured-crews.csv), 200 replicates, $k{=}30$, $H{=}\$10$, noise $0.15^2$ all match figures.py; 16 unknowns (6 workers + 4 checkers + 6 planners) behind 138 crews (one same-vendor worker/checker pair removes 6 of 144) match `role_truth`. (iv) *Row fields*: paper :199–201 = `record.Row` (record.go:106–116) = `schema.js` payload, nonce on the batch not the row (:201; outbox envelope, `X-Codeaf-Install`). (v) *Status* (:550) against the tree — see defect 4. (vi) README and the settings hint — see new defect 8. + +## The seventeen defects and the overstated claims + +1. **Prop 4, capped counts — fixed.** One capped external $W=K\ell/(K+\ell)$, $K=k+\tilde n$, split $k/K$ and $\tilde n/K$ (:331–333, proof :338); the "at most $\ell$ own rows" sentence now scopes to catalog and pool together (:345). Derivation verified, including the $b$ term and the $\sigma^2+\tau_1^2$ pool variance the first report found missing. +2. **Drift $\tfrac12$ — fixed.** (13) at :264 carries the matching $\sigma^2(1/n_d+1/n_{d-1})$ noise term; no factor of one half. +3. **Additive sheet vs max-join — partly fixed.** §5.2 (:224) now names both stores and the own sheet's description matches `tally.Merge` (tally.go:227) and its cell key (tally.go:123); but "The relay's store is the sheet of Definition 2, keyed by install and day and merged by (11)" is false of the code — the relay folds rows additively (relay/src/worker.js:160) and the join has no caller outside its test — see new defect 1. The abstract's join-semilattice sentence (:64) inherits the gap. +4. **Status vs tree — fixed.** The judge hook is wired to chat landings (cmd/codeaf/poolrecord.go:46–66, the `session.TaskLanding` seam), rows are recorded into the own sheet and the outbox and pushed (record.go, cmd/codeaf/poolinstall.go:71–99), the built-in key is compiled in (cmd/codeaf/pool.go:31), the blend is the Corollary's at $k{=}30$, and the relay and mirror exist. One word overstates: "the join" in the relay clause — new defect 1. +5. **Relay code — fixed.** `relay/` lands with `src/{worker,schema,sheet,index}.js`, tests and `verify.mjs`; `wrangler.toml` binds the `/pool/*` route. Residual: the ten-step runbook is at `docs/design/model-pool/RUNBOOK.md`, not "in that directory" (:493) — new defect 6. +6. **Public key — fixed.** `poolPublicKeys` carries the signer's key (cmd/codeaf/pool.go:31, matching relay/wrangler.toml's `POOL_PUBLIC_KEY`), a stored key overrides it (`poolTrustedKeys`, cmd/codeaf/pool.go:44–53), and the background refresh starts under it (cmd/codeaf/poolindex.go:75–81). +7. **Run counts — partly fixed.** The abstract no longer quotes a number (:64); Figure 6's "14 cells from 120 scored runs" (:478) is the defensible reading (data/seed-cells.csv: 14 cells, $n$ sums to 225, about 120 judged runs at ~1.9 scored roles). But the new number, "fitted by a ridge model on 72 scored runs over nine tasks" (:186, :442), is unverifiable on this branch: the runs column of measured-crews.csv sums to 1224, and `figures.py` reads the fitted columns without fitting anything, so nothing in the repository produces 72 or nine. +8. **Role-level ratio — fixed.** :445 and :461 state both comparisons and every number matches: 0.54 clean bit, 0.31 mean score, 0.15 role at $N{=}400$; "half" and "a third" hold. +9. **Prop 3 — partly fixed.** The statement is corrected (:243: one constant and one constraint per component; when $G$ is connected the single constraint identifies everything), but the proof still ends "With the single centring constraint, exactly one component is fully identified" (:246) — the removed falsehood, still false for $c\ge2$ (one constraint leaves a $(c{-}1)$-dimensional gauge; no component is pinned) — new defect 2. +10. **Subset of crews — fixed.** "all 23 chat-door crews" (:445); `figures.py` `fig_regret` takes every `door=="task"` row, and the CSV holds 23; the `[:12]` slice is gone. +11. **`min_installs` — partly fixed.** The relay computes and publishes a distinct-install count per cell (relay/src/sheet.js `aggregate` → `installs`; relay/src/index.js `buildIndex`), and §5.4's robustness claim is reduced to "a minority of installs that are broken or adversarial" (:253). But "A cell backed by fewer than three installs is published but not read" is still false of the code: both readers floor on the observation count (internal/pool/index/index.go:265 `n < x.minInstalls`; crewpick.go:235 `c.N < minInstalls`), `index.Cell` never reads a cell's `installs` field, and the seed ships `min_installs: 1` with no `installs` fields (internal/pool/index/seed.json:4). §9's flood sentence survives unchanged (:499) — the response says it was reduced; only the §5.4 half was. +12. **Row fields — fixed.** Row (role, model, score, judge, door, size, day) (:199–201) = `record.Row` (record.go:106–116) = `schema.js`'s validated payload; nonce drawn once per profile, on the batch header not the row (:201; outbox.go, cmd/codeaf/poolinstall.go:44–69); no price or duration anywhere in a row; Definition 2's cell (:206), the relay's `cellKey` (sheet.js:24) and §9's "one triple per (install, cell, day)" (:493) now spell the cell one way. +13. **Abstract — fixed.** The constants sentence scopes itself to statistical constants and names the two user prices (:64), matching Table 2 up to its last clause (new defect 5); the modes read as the code behaves (off reads nothing: cmd/codeaf/poolindex.go `poolIndexFor`). +14. **`explore` — fixed.** "specified, not on this head" (:542); no such key exists in internal/config, consistently. +15. **$E$ as surrogate — fixed.** :176 derives (3) as (2) at $r=1$, states the regime, and allows (2) directly where a delivered rate is measured. +16. **Imputation — fixed.** :110 describes the implemented rule (crewpick.go `fill`: per present index, median ratio over at least three donors, offers averaged, capped at the pool's maximum; scale-proportional standing below three donors). +17. **Loader — fixed.** :490 "the fresher of the cached index and the embedded seed index, by generated day" matches `index.Fallback` (index.go:96–104). + +Overstated claims: + +- **Abstract privacy — fixed** (:64 states what a row contains, not what can be inferred). +- **Prop 5 — fixed** (product posterior; the proof names the ignored dependence and scopes exactness to the product approximation, :424). +- **Exploration tax — partly fixed** (:445). Thompson is no longer the worst row (greedy is), but "no better than the table" is contradicted by `tab-regret.tex` at $N{=}10$ (0.97 vs 1.01) and $N{=}40$ (0.93 vs 0.95); only at $N{=}20$ is Thompson worse (0.97 vs 0.95). The differences are small; the sentence should say "within noise of the table" or quote the numbers. +- **figures.py — partly fixed** (:482). The constants at its head are $H$, $\lambda$, $K$, REPS; fig-shrink's catalog 80, measured 65 and $\sigma{=}15$, and fig-front's iso-line anchors, are function-local, so "from the two CSV files it reads and the constants written at its head" still overstates two figures. +- **pool status/show — partly fixed** (:546). `pool show` matches the code (each address with its source, the index summary); status counts pending, but "says whether the relay answered" is not implemented — status never reaches the network and prints the mode-derived can-send/can-read (cmd/codeaf/pool.go:213). New defect 7. +- **Rollback — fixed in part** (:496). The version refusal is real (internal/pool/pull/pull.go `good`); the join clause leans on the unwired join — new defect 1. +- **Judge floor — fixed** (:195, absolute; judge.go:45, applied absolutely in `Pick`). +- **"cents per run" — fixed** (:195, "one short completion per role"). + +## New defects + +1. **§5.2 (:224) and §9 (:496) claim the relay's store merges by the join (11); it does not.** The worker folds each received row additively into the per-(install, day, cell) triple (`fold`, relay/src/worker.js:160; relay/src/sheet.js:34–42), and `join` (relay/src/sheet.js:35) is called by no file except relay/test/sheet.test.js. The outbox marks a batch sent only on a 2xx (internal/pool/outbox/outbox.go, `sendHTTP`), so a batch whose response is lost after the server folded it is re-sent and folded twice: Proposition 2(iii)'s "with any duplication" and "the join of Section 5.2 makes a replayed old publication harmless" do not hold of the wire as shipped, and Status's word "the join" (:550) overstates with them. Fix: have `POST /v1/rows` carry the sender's running totals per cell and merge by `join`, or add batch-level idempotency. +2. **Proof of Proposition 3 (:246), last sentence.** "With the single centring constraint, exactly one component is fully identified" is false when $G$ has $c\ge2$ components: the constraint $\sum_C w_C\delta_C=0$ (weights $w_C$ = judge mass per component) leaves a $(c{-}1)$-dimensional gauge, so *no* component's levels are pinned under a single constraint. Delete the sentence or write what the statement's last line already says. +3. **"The relay centres $\beta$ within each component" (:249).** relay/src/sheet.js's `judgeSeverity` (:57) re-centres $\beta$ by one global weighted mean each sweep (:76–86), so the sentence does not describe the shipped fit, and on a disconnected judge–cell graph the alternating fit's fixed point remains initialisation-dependent — the first report's note, still true of the code. +4. **§9's hourly job and Figure 2 (:286, :493).** The scheduled job runs the judge-effect fit and the Huber aggregation and no drift estimate (relay/src/sheet.js; Status :550 itself defers (13)), yet §9 says the job "runs the judge-effect, robust-aggregation and drift estimates" and the figure's relay node prints "drift (13)". The mirroring is not the job's either: it is a separate GitHub workflow on a six-hour cron (.github/workflows/pool-mirror.yml:11), while the relay's own cron is hourly (relay/wrangler.toml:17). +5. **§6 (:355) and Table 2's caption (:391).** "These are published in the index beside the cells ... re-estimated at every relay cycle", and the ten-degrees-of-freedom withholding rule, are not on the branch: the relay's index document carries only schema, version, generated, min_installs, judges, rubrics, aliases, metrics and cells (relay/src/index.js `buildIndex`), no estimator lives in relay/src, and Status defers the moment estimators (:550). The caption also counts $v_r$ — measured from token accounting, not estimated — among "the rest are empirical-Bayes estimates". State these as specified-not-landed, as Status does. +6. **§9 (:493): "the runbook in that directory is ten steps."** The runbook has ten steps (docs/design/model-pool/RUNBOOK.md, Deploy 1–10) but lives beside this document, not in `relay/`; relay/README.md carries no steps. +7. **§10 (:546): "pool status counts the rows waiting in the outbox and says whether the relay answered."** The count is real (cmd/codeaf/pool.go:213, `pending %d`), but nothing in status contacts the relay; can-send/can-read are the mode's permissions (cfg.CanSend/CanRead), not an answer. Only `pool verify` reaches the network. +8. **README vs the paper on what leaves an install.** The README's Model Pool section enumerates "(role, model, a number, a day)" (README.md:201–203); the paper's row also carries judge, door and size (:199–201). The settings hint avoids the enumeration and is consistent (internal/tui3/settings.go:680–683). Name the row's fields in the README or point at the paper's list, so the two surfaces say the same thing. + +## For a camera-ready version + +- Make the relay's write path match §5.2 — snapshot triples merged by the join, or batch ids — so Proposition 2(iii) holds of the wire and Status's "the join" stands. +- Delete Proposition 3's proof-final sentence; implement per-component centring in the relay's fit or move the sentence to the design. +- Land or reword the four present-tense relay claims: drift in the hourly job (§9, :286), the constants and ten-DOF rule (§6, Table 2 caption), pool status's relay line (§10). +- Carry the distinct-install count through the readers (an `installs` field on `index.Cell`, the crewpick floor) so §5.4's three-install sentence is true of the code; revisit §9's flood sentence once fabricated installs can be bounded. +- Give "72 scored runs over nine tasks" a checkable home (state how the CSV's runs column, 1224 over 56 crews, relates to it, and that `rows[1]` — ds-v4.1-flash with qwen3.8-max — is the shipped balanced pick), and make §5.6's $\hat b=-15.7$, 75 and 231 reproducible from `figures.py` and the committed CSVs, which today carry no catalog indexes. +- Align the README's field list with the row's seven fields. diff --git a/docs/design/model-pool/REVIEW-RESPONSE.md b/docs/design/model-pool/REVIEW-RESPONSE.md new file mode 100644 index 0000000000..af8e3e7221 --- /dev/null +++ b/docs/design/model-pool/REVIEW-RESPONSE.md @@ -0,0 +1,54 @@ +# Response to the referee + +The report in `REVIEW.md` was written against the draft at the head before this one. The paper was `learn.tex` then and is `pareto-crewing.tex` now; each numbered defect is answered below with what changed; the section numbers are the revised document's. + +## Defects + +1. **Proposition 4, the split of the cap.** Corrected. The external count is now one capped quantity `W = Kℓ/(K+ℓ)` with `K = k + ñ`, `ñ = n_pool·ℓ/(ℓ+1)`, shared between catalog and pool as `k/K` and `ñ/K` of `W`; the proof integrates `μ_rm` out once and states the precision-weighted mean. The corollary's statement ("worth at most ℓ own rows") now applies to catalog and pool together, and the sentence that follows says which form the shipped pick is (`ℓ = ∞`, `b = 0`). +2. **Drift estimator.** The factor ½ is gone from (13); the difference of adjacent day means has variance `ω² + σ²(1/n_d + 1/n_{d-1})`. +3. **Additive local sheet versus max-join.** Section 5.2 now names both stores: the install's own sheet (`internal/pool/tally`) merges by addition because each row is folded once; the relay's store is keyed by (install, day, cell) and merges by the join. +4. **Status paragraph.** Rewritten for the head this document ships with (judge hook, install nonce, push, relay, mirror) and lists what is specified but not landed. +5. **Relay code.** `relay/` lands on this head; Section 9 describes it as present. +6. **Public key.** Compiled into the binary on this head; the private-relay setting overrides it. +7. **Run counts.** The crew table is "56 combinations fitted by a ridge model on 72 scored runs over nine tasks" (caption of Figure 1 and Section 8); the seed cells come from the ledger's 120 runs (Figure 6). The abstract no longer quotes a number. +8. **Role-level ratio.** Section 8 and the abstract now state both comparisons: the role-level learner sits at half the crew-level learner fed the same scores and a third of the one fed the clean bit. +9. **Proposition 3.** Restated: identified up to one constant per connected component, one centring constraint per component; proof names the kernel. +10. **Subset of crews.** The regret experiment now runs on all 23 chat-door crews; the table and figure were regenerated and the text quotes the new numbers. +11. **`min_installs`.** Section 5.4 says the relay publishes a distinct-install count per cell and that the reader's floor applies to it; the flood claim is reduced to what the design bounds (a minority of broken or hostile installs) and says what it does not (fabricated majorities). +12. **Row fields.** The row is (role, model, score, judge, door, size, day); the install nonce is on the batch; dollars and minutes are stated to stay in the install's ledger. The cell is spelled the same way in Definition 2, the join and the row. +13. **Abstract.** The constants sentence says what the user still sets (the price of a defect and of time); "stay out and still learn" is replaced by the three modes as they behave. +14. **`explore`.** Marked "specified, not on this head" in the settings table. +15. **`E` as a surrogate.** Section 4 states that (3) is `E_H` at `r = 1`, when the ranking agrees, and that the picker can use `E_H` directly where a delivered rate is measured. +16. **Imputation.** Section 3 describes the implemented rule (median ratio over at least three donors per present index, averaged, capped at the pool's maximum, scale-proportional standing below three donors). +17. **Loader.** Section 9 says the fresher of cache and seed is read, by generated day. + +## Overstated claims + +- Privacy in the abstract is now a statement of what a row contains, not of what can be inferred. +- Proposition 5 is stated for the product posterior and says which dependence the draw ignores. +- The exploration tax is stated as forty runs in which Thompson is no better than the table. +- `figures.py` is said to reproduce the figures from the CSVs and the constants at its head. +- `pool status` and `pool show` are described as they print. +- Rollback is caught by the version; the join makes a replayed old publication harmless. +- The judge floor is stated as absolute. +- "Cents per run" is replaced by "one short completion per role". + +## What the referee asked for and is not in this revision + +- A catalog bias term `b` is added to the hierarchy with its estimator, and the seed's own `b̂ = −13.7`, `τ̂₀² = 0` are reported with the reason the day-one `k = 30` stands. +- Experiments on judge effects, drift, the settled probability and the pair term, and a flood analysis, remain future work; the Status paragraph lists them among what is specified but not landed. + +## Second round (`REVIEW-2.md`) + +Verdict was minor revision; the eight new defects are answered here. + +1. The relay folds rows additively; the paper said the join. §5.2 now says which merge is which: addition within a store (own sheet, relay), the join between copies of sheets, and names the retry double-count and the quota that bounds it. Status and §9 no longer say the relay's store is join-merged. +2. Proposition 3's proof: the sentence the referee quotes is not in the revised proof; the proof ends at the kernel and the per-component constraint. +3. §5.3 now says the reference relay centres by one global weighted mean and that per-component levels are a convention on a disconnected graph. +4. Drift is out of §9's job description and both figures' relay nodes; it stays in Status as next to land. +5. §6 now states the constants' publication and the ten-degrees-of-freedom rule as the design, and that this head's index carries the cells and the day-one constants. +6. "the runbook beside this document". +7. `pool status` reaches the relay on this head (client B, after the referee's clone); the sentence stands. +8. README's field list now names the row's seven fields. + +Camera-ready items not taken here: join-merged relay writes or batch ids; per-component centring in the relay; the distinct-install count through the readers; a checkable home for the 72 runs; the seed's offset numbers computed by `figures.py`. diff --git a/docs/design/model-pool/REVIEW.md b/docs/design/model-pool/REVIEW.md new file mode 100644 index 0000000000..b91c6fa63d --- /dev/null +++ b/docs/design/model-pool/REVIEW.md @@ -0,0 +1,67 @@ +# Review of docs/design/model-pool/learn.tex + +## Summary + +The paper builds the crew picker in three layers: a catalog-only pick (a pareto front in bill and quality with a log-scale knee), a privacy-preserving measurement pool that replaces the catalog's proxy for quality, and a Thompson-sampling learner over role-level posteriors. The first layer is implemented and matches the prose in every particular checked: (1) is `crewpick.SeatCost` (crewpick.go:258), (2) is `quality` (crewpick.go:463) with the same pool-normalised scale, (3) is the shortlist at `Floor = 0.80` (crewpick.go:183, 454), (4)–(5) are `FrontWith`/`Presets`/`AtKnob`/`knee` (crewpick.go:315, 376, 388, 407) including the id tie order and the epsilon, the seat shapes and the vendor rule match `DefaultShapes` and the crew loop, and the Corollary's blend with `k = 30` is `blendQuality` with `PriorWeightAt = 30` (crewpick.go:479, 189). The judge rule of 5.1 matches `judge.Pick` (judge.go:159) with floor 60 (judge.go:45), the pool modes match `poolcfg`, the signed pull with version refusal matches `internal/pool/pull`, and the seed index matches Figure 3's construction exactly: its 14 cells are Beta(1,1) posterior means and sds of a no-major-defect bit times 100 (verified against `internal/pool/index/seed.json` and `data/seed-cells.csv`, e.g. worker/glm-5.3-flash n=62, mean 65.6, sd 5.9). Lemmas 1 and 2, Propositions 1, 2 and 5, the ICC formula of 5.4, estimators (17) and (18), and the steady-state Kalman relation of 5.5 are correct as mathematics. The central statistical claim is not: the capped counts of Proposition 4 are not the posterior its own proof derives, the drift estimator (14) returns half of what it estimates, and Proposition 3's closing sentence is false. Against the branch, the Status section overstates the wiring in six places, and two of the quoted experiment numbers do not match the committed tables. Major revision. + +## Defects, by severity + +1. **Proposition 4, eq (15)–(16) (learn.tex:325–335). The capped counts are not the posterior the proof derives.** The proof computes the catalog-and-pool precision about $\mu^u$ as $(k+n^{\mathrm{pool}})\ell/(k+n^{\mathrm{pool}}+\ell)$ and then claims this "splits into the two terms of (16) in proportion to $k$ and $n^{\mathrm{pool}}$". It does not: $k\ell/(k+\ell) + n\ell/(n+\ell) \ne (k+n)\ell/(k+n+\ell)$ for any positive $k, n, \ell$. The correct split under the proof's own model is $k\ell/(k+n^{\mathrm{pool}}+\ell)$ on the catalog and $n^{\mathrm{pool}}\ell/(k+n^{\mathrm{pool}}+\ell)$ on the pool. With $k=30$, $n^{\mathrm{pool}}=100$, $\ell=30$ the printed weights sum to 38.1 against the proof's 24.4, and as $k$ and $n^{\mathrm{pool}}$ grow the printed formula lets catalog and pool together approach $2\ell$ own rows, breaking the proof's own bound of $\ell$ and the text's claim after (16) that the pool's evidence is "worth at most $\ell$ own rows" (true only of the sum). A second error sits in the proof's first stage: precision $(k+n^{\mathrm{pool}})/\sigma^2$ treats pool rows as $\sigma^2$-noise observations of $\mu_{rm}$, but under the hierarchy (6) they are other installs' rows with marginal variance $\sigma^2+\tau_1^2$, so their contribution is $n^{\mathrm{pool}}/(\sigma^2+\tau_1^2)$ — the same inconsistency biases (19). Fix: state the posterior precision as one external term, $W = K\ell/(K+\ell)$ with $K = k + n^{\mathrm{pool}}\sigma^2/(\sigma^2+\tau_1^2)$ (or $K = k+n^{\mathrm{pool}}$ under the stated simplification), weight the catalog and pool as $k/K$ and $n^{\mathrm{pool}}/K$ of $W$, keep "external evidence $\le \ell$" for the sum, and make (19)'s noise term match. The Corollary is unaffected (it is the $\ell=\infty$ limit, which both forms share). + +2. **Eq (14) (learn.tex:264). The drift estimator carries a spurious factor of one half.** For adjacent-day means, $\operatorname{Var}(\bar y_d - \bar y_{d-1}) = \omega^2 + \sigma^2(1/n_d + 1/n_{d-1})$; the bracket subtracts exactly that noise, so (14) returns $\hat\omega^2/2$ and the filter forgets at half the rate the data show. The $\tfrac12$ is correct only for differences taken at lag two. Fix: drop it, or difference at lag 2 with the matching noise term. The steady-state relation in the same paragraph, $\alpha^2/(1-\alpha) = \bar n\,\omega^2/\sigma^2$, is correct. + +3. **Eq (13)/Proposition 2 (learn.tex:209–219) against `internal/pool/tally/tally.go`: the shipped sheet does not merge by the join.** `Sheet.Merge` adds ("adds other into s", tally.go:201; "Merging a sheet into itself doubles it", tally.go:199) — commutative and associative but not idempotent — and the cell address (metric, role, model, dims) carries no install key (tally.go:123). Proposition 2 itself is correct mathematics for per-install running totals; addition is safe only for pairwise-disjoint sheets, so a re-delivered or duplicated publication double-counts and claim (iii)'s "with any duplication" does not hold of the shipped merge. The abstract's "sufficient statistics that form a join-semilattice, so any number of installs and a relay can merge them in any order and reach the same state" is a design, not a description of anything on the branch, and the Status paragraph (learn.tex:548) puts "the outbox and the sheet" on the branch. Fix: either say the shipped sheet is additive and duplicate safety is the outbox's per-node sent markers (a guarantee that does not federate), or land the install-keyed store with the max-join before making the federation claims. + +4. **Status (learn.tex:548). The judge, the sheet and the outbox are on the branch as libraries with no call sites.** `internal/pool/judge` is imported by nothing (no caller of `judge.Judge` or `judge.Pick` outside the package), `internal/pool/tally` is imported by nothing, and `outbox.Append` has no caller — `cmd/codeaf/pool.go` only reads `Pending`. No run is judged, no row is written, no sheet accrues, so the closing sentence "Until the relay exists the pool is the seed index plus each install's own sheet" is false: on the branch the pool is the seed or cached index alone, blended through `config.autoPrior` (internal/config/auto.go:144). Fix: say the three packages exist unwired and that *learn* on the branch means catalog plus index. + +5. **"The relay's code is in this repository" (learn.tex:544, with the deploy sentence at 492). There is no `relay/` directory.** Status (learn.tex:548) itself lists the relay as "specified here and next to land"; sections 9 and "What the user controls" contradict it in the present tense. Fix: future tense, or land the directory. + +6. **"The signing key's public half is compiled into the binary" (learn.tex:495, 544). It is not.** `var poolPublicKeys []ed25519.PublicKey` is empty, with the comment "EMPTY ON THIS BRANCH" (cmd/codeaf/pool.go:37); `pool verify` refuses without `--key` (cmd/codeaf/pool.go, `verifyPool`), and the background refresh never starts because `startPoolIndexRefresh` returns when the key list is empty (cmd/codeaf/poolindex.go:77) — so "a refresh runs in the background" (learn.tex:489) is inert on this branch. Fix: say the mechanism is in place and the key ships beside the first published index, or compile a key. + +7. **"120 scored runs" (abstract, learn.tex:64; section 8, learn.tex:441; captions at 186 and 477). The number matches neither committed dataset where it is used.** The `runs` column of `data/measured-crews.csv` sums to 1224 across the 56 crews, and the seed's 14 cells carry n summing to 225 role-level scores (consistent with about 120 judged runs at 1.9 roles each — Figure 3's usage is the defensible one). But the abstract, the Figure 1 caption and section 8 attach the 120 to the 56-crew table, whose rows are backed by 1224 runs, and the section 8 experiments take that table as ground truth. Fix: say what the 120 counts (judged runs) and correct the three locations that attach it to the fitted-crew table. + +8. **Section 8 (learn.tex:460). "at four hundred runs the role-level learner's regret is about a third of the crew-level learner's on the same scores."** `tab-semibandit.tex` at $N=400$ reads 0.15 (role), 0.31 (crew, mean judge score), 0.54 (crew, clean bit). Against the learner the sentence names — fed the same scores — the ratio is about one half. A third holds only against the clean-bit learner, which is not "on the same scores"; the abstract's "factor of three" matches that other comparison. Fix: name the baseline and quote it; as written the sentence is wrong against its own table. + +9. **Proposition 3 (learn.tex:246). "With the single centring constraint, exactly one component is fully identified."** False when $G$ has $c \ge 2$ components: the kernel of the two-way design is $c$-dimensional and the single constraint $\sum_j n_{\cdot\cdot j}\beta_j = 0$ leaves an $(c{-}1)$-dimensional gauge that shifts $\mu$ in every component jointly (shifts weighted by $\sum_C N_C\delta_C = 0$), so no component's levels are pinned — only the within-component contrasts, as the preceding sentence correctly says. Identifying a component's levels needs one centring constraint per component. Note also that the alternating re-centring (learn.tex:235–239) is global, so on a disconnected $G$ its fixed point depends on the initialisation. Fix: one constraint per component, or state that identification holds only when $G$ is connected. + +10. **Section 8 (learn.tex:444). "the twelve chat-door crews" — the data has 23; `figures.py` keeps the first 12 in file order (figures.py:136, `[:12]`).** No criterion for the twelve is stated anywhere; which twelve they are sets the oracle ($1.94 all-in) and every number in `tab-regret.tex`. Fix: state the subset rule, or use all 23. + +11. **Section 5.4 (learn.tex:253). "A cell backed by fewer than three installs is published but not read" — nothing on the branch counts installs.** The index reader drops a cell when its n — an observation count — falls below the document's `min_installs` (internal/pool/index/index.go:265), `crewpick.PriorFromCells` applies the same test to a count (crewpick.go:231), and the shipped seed sets `min_installs` to 1 (internal/pool/index/seed.json:4); one install contributing three rows passes. The related claim in section 9 (learn.tex:498) that "the min-installs and robust-aggregation rules bound what a flood of fabricated nonces can move" is unsupported: the Huber rule's breakdown point of 1/2 is in installs, fabricated installs are unauthenticated, and a flood that outnumbers genuine installs passes the breakdown point while each nonce clears any rows-per-install cap at trivial cost. Fix: carry a distinct-install count per cell in the published document, set the shipped floor to it, and either analyse the flood bound or drop the claim. + +12. **Section 5.1 (learn.tex:199–201). The row fields exist nowhere; the shipped row is an envelope.** The outbox stores `{schema, day, nonce, payload}` with a fresh 16-byte nonce per row (internal/pool/outbox/outbox.go:49, 135) and an opaque payload that nothing builds; the per-profile install nonce of learn.tex:201 exists in no file; the sheet's cell is (metric, role, model, dims) with the triple (n, sum, sumsq) (internal/pool/tally/tally.go:123, 266), which can carry judge/day/size/door as dims but carries none today. Relatedly, Definition 2 (learn.tex:203) puts day inside the cell and section 5.5 says "The day is a cell dimension", while section 9 (learn.tex:492) says the store holds "one triple per (install, cell, day)" — the two spellings of cell differ. Fix: present the row schema as specified and add it to Status, and pick one spelling of the cell. + +13. **Abstract (learn.tex:64), two claims contradicted by the paper's own later sections.** "every constant except the user's price of a defect is an empirical-Bayes estimate" — Table 2 says $\lambda$ is the user's, $\phi=0.8$ and the judge floor 60 are structural, and $v_r$ is measured; only $\sigma$, $k$, $\ell_f$, $\omega^2$ and $\beta$ are estimated. "stay out, and still learns from its own runs" — with the pool off the code reads no index at all (cmd/codeaf/poolindex.go:50) and the paper itself says "with it off, only the catalog prior remains" (learn.tex:544). Fix both sentences. + +14. **"What the user controls" (learn.tex:540). The `explore` setting does not exist.** `internal/config` carries `model_pool` (settings.go:119), the pick words and the crew presets, but no `explore` key anywhere; nothing toggles a draw. Fix: mark the row "next to land", as Status does for the draw. + +15. **Section 4 (learn.tex:171–173). The working objective $E$ is neither derived form.** $E = c+\lambda t + H(1-p_{\mathrm{clean}})$ equals $E_H$ with $r$ set to 1 and $p$ replaced by $p_{\mathrm{clean}}$: non-deliveries are priced as shipped defects, and the rerun term $(1-r)(c_f+\lambda t_f)/r$ — and $E_0$'s $1/(rp)$ factor — are gone. For a cheap but flaky crew with $r$ well below 1, $E$ can rank it above what $E_0$ or $E_H$ would; the paper never states the surrogate. Fix: one sentence deriving $E$ as $E_H$ under $r=1$ (or adding the rerun term), and the regime where the surrogate is safe ($H$ large next to run cost, $r$ not far below 1). + +16. **Section 3 (learn.tex:110). The imputation sentence does not describe the implemented imputation.** `crewpick` uses the donor median only with three or more donors, averages the offers from every present index, caps the estimate at the pool's largest measured value of the missing index, and falls back to scale-proportional standing below three donors (crewpick.go:525, 582). The sentence describes none of this. Fix: describe the implemented rule, or mark the sentence as simplified. + +17. **Section 9 (learn.tex:489). "The loader reads the cached index if one is present and the embedded seed index otherwise."** The cache wins only when it is not older than the seed: `index.Fallback` compares the generated days and prefers the seed on an older cache (internal/pool/index/index.go:96–104). Fix: "the fresher of the cache and the seed by generated day." + +## Correct but overstated + +- Abstract (learn.tex:64): "nothing an install sends can be read back into what it was working on." Absence of text is not unlinkability: a row's (dollars, minutes, size, door, day, role, model) is a quasi-identifying tuple against anyone who knows the task schedule. Section 9's store-scoped claim (learn.tex:498) is the defensible form; the abstract goes further than it. +- Proposition 5 (learn.tex:420): "is exactly Thompson sampling over the crews' posterior costs." Exact only if the per-(role, model) posteriors are independent; 5.3's judge-effect fit and 5.5's drift model both induce cross-cell dependence that the independent draw ignores. It is Thompson sampling on a product approximation to the posterior. +- Section 8 (learn.tex:444): "Thompson pays a small exploration tax in the first twenty runs." `tab-regret.tex` puts Thompson at 0.98 and 0.97 at $N=10$ and 20 — the worst row of the table, above even the fixed table (0.91, 0.93). The tax is real; "small" is not. +- Section 8 (learn.tex:481): "reproduces every figure and table from the two CSV files it reads." Figure 5's constants (catalog 80, measured 65, $\sigma=15$) and Figure 1's iso-line anchors are hardcoded in `figures.py`. +- Sections 9 and "What the user controls" (learn.tex:489, 544): `pool status` prints a pending count ("pending %d", cmd/codeaf/pool.go:172), not the rows; `pool show` prints the index's generated day and counts, not its cells. +- Section 9 (learn.tex:495): "(Section 5.2's join makes a rollback detectable)." A rollback is detected by the version number the puller compares (`Puller.good`, internal/pool/pull/pull.go); the join makes old data harmless, not detectable. +- Section 5.1 (learn.tex:195): "The floor exists because a judge below the worker's class rubber-stamps." The rule is an absolute floor of 60 (judge.go:45), not a floor at the worker's class: a coding-61 judge over a coding-90 worker passes it. The rationale is relative, the rule absolute. +- Section 5.1 (learn.tex:195): "at present prices it is cents per run" — not checkable from anything in the repository. + +## What a reader would still want + +- The posterior under the full hierarchy (pool rows at variance $\sigma^2+\tau_1^2$): the corrected (16) and (19), and a first $\hat k$ computed from the seed's own 14 cells — the paper's data can produce one and never does. +- The regime in which (7) ranks crews as $E_0$ and $E_H$ do: a worked example with $r<1$ and a cheap, flaky crew. +- Experiments for the untested sections: the judge-effect fit of 5.3 (the seed carries a single judge id, so the shipped data cannot exercise it), the drift model of 5.5, and the settled probability $\pi^\ast$ of section 7 — no trajectory is shown for any of them. +- The pair and interaction term of 6.1: specified with an estimator, never simulated. +- The subset rule behind "the twelve chat-door crews" and the identity of the "fixed table row" (the second line of `data/measured-crews.csv`: glm-5.3-flash with qwen3.8-max, all-in regret \$0.96 against the subset oracle), with `tab-regret.tex`'s sensitivity to both choices. +- The fabricated-install flood analysed rather than asserted (defect 11): what the Huber rule bounds once fabricated installs are the majority, and the cost per nonce to get there. +- Instance-dependent gaps: the paper's own caption says a dozen crews sit within cents; the whole regret scale (\$0.96 for the table against a \$1.94 oracle all-in) is governed by gaps the paper never reports, so "a third" and "factor of three" are dataset-specific with no gap structure given. +- Where the day-one constants come from: $\sigma=15$ and $\ell_f=30$ in Table 2 have no derivation. + +## Verdict + +Major revision. The catalog half of the paper is sound and matches the code line for line, and the CRDT, fixed-effects and robustness designs are right as designs. But the central estimator is wrong as printed (defect 1), one moment estimator is off by a factor of two (defect 2), one proposition's closing claim is false (defect 9), and the paper's account of what is implemented overstates the branch in six places (defects 3–6, 12, 14). The experiments support the direction of every claim they are attached to, but two quoted numbers do not match the committed tables (defects 7–8). Fix the mathematics, reconcile the Status paragraph with the tree, and re-derive or re-run the affected numbers; the architecture is worth publishing once every claim is as accurate as the catalog section already is. diff --git a/docs/design/model-pool/RUNBOOK.md b/docs/design/model-pool/RUNBOOK.md new file mode 100644 index 0000000000..51ba8fa2ea --- /dev/null +++ b/docs/design/model-pool/RUNBOOK.md @@ -0,0 +1,108 @@ +# Model Pool relay — runbook + +The relay is a Cloudflare Worker under `relay/`. It accepts measurement rows +from installs over NDJSON at `POST /pool/v1/rows`, folds each install's rows +into per-install running totals ("sheets") in KV, and once an hour publishes a +signed, versioned JSON index that installs read. It serves, all under the +`/pool/` prefix: + +- `POST /pool/v1/rows` — submit rows. Requires the `X-Codeaf-Install: <32 + lowercase hex>` header; the body is at most 256 KiB and at most 200 lines. +- `GET /pool/index.json` — the published index document. +- `GET /pool/index.json.sig` — its detached Ed25519 signature. +- `GET /pool/healthz` — a liveness probe. + +Every other path answers 404 with an empty body. The document is signed with +Ed25519: the Worker holds the 32-byte private seed as the secret +`POOL_SIGNING_KEY` (standard base64) and the 32-byte public key as the plain var +`POOL_PUBLIC_KEY` (standard base64). Installs read +`https://codeaf.agentfield.ai/pool` by default; `CODEAF_MODEL_POOL_RELAY_URL` +points an install elsewhere and `CODEAF_MODEL_POOL_PUBLIC_KEY` sets the trusted +key. + +## Deploy + +1. Have a Cloudflare account with **Workers & Pages** enabled. +2. `npm i -g wrangler && wrangler login`. +3. `cd relay && wrangler kv namespace create POOL`, then paste the returned id + into the `[[kv_namespaces]] id` in `wrangler.toml`. +4. Generate the signing key once, off the repository, and set the secret: + - `openssl genpkey -algorithm ed25519 -out pool-ed25519.pem` + - raw 32-byte seed, standard base64: + `openssl pkey -in pool-ed25519.pem -outform DER | tail -c 32 | base64` + - raw 32-byte public key, standard base64: + `openssl pkey -in pool-ed25519.pem -pubout -outform DER | tail -c 32 | base64` + - `wrangler secret put POOL_SIGNING_KEY` reads the seed from stdin, so pipe + it in: + `openssl pkey -in pool-ed25519.pem -outform DER | tail -c 32 | base64 | wrangler secret put POOL_SIGNING_KEY` + - put the public key into `[vars] POOL_PUBLIC_KEY` in `wrangler.toml`. + Never write the seed anywhere in the repository. +5. `wrangler deploy`. +6. DNS and the route: create a proxied DNS record for `codeaf.agentfield.ai` + (AAAA `100::` or A `192.0.2.1`, proxied) so the route can attach. The route + in `wrangler.toml` binds `codeaf.agentfield.ai/pool/*` to the Worker. + Installs read `https://codeaf.agentfield.ai/pool` by default, and + `CODEAF_MODEL_POOL_RELAY_URL` points an install elsewhere. +7. First publication: `wrangler triggers deploy` and wait for the next `:17`, + or, under `wrangler dev --test-scheduled`, drive it by hand with + `curl -X POST "http://localhost:8787/__scheduled"`. Publishing does not + depend on the cron: the first read past PUBLISH_EVERY republishes in the + background while serving what is stored, so the cron is a floor rather + than a requirement, and step 7a below is only for a store you want filled + before anyone reads. +7a. If the store is still empty forty minutes after the first `:17` (a fresh + Worker's cron can miss its first tick), trigger the scheduled handler once + by hand: `wrangler dev --test-scheduled` bound to the remote namespace, then + `curl "http://localhost:8787/__scheduled?cron=17+*+*+*+*"`; the next `:17` + takes over from there. The same hand trigger is how to publish at once + after loading rows (`relay/tools/prime.py`, or any large batch): a read + republishes only when the stored document is older than PUBLISH_EVERY, so + fresh rows behind a young document wait up to an hour otherwise. +8. Verify from a laptop: `curl -sI https://codeaf.agentfield.ai/pool/index.json` + (expect 200 and an ETag), then `codeaf pool verify`, then `codeaf pool + status`. +9. Know what the store holds — one running total per install, cell and day; a + per-install per-day quota counter; and the last index document and its + signature — and how to wipe it: `wrangler kv key list` to see the keys and + `wrangler kv key delete ` to drop one. +10. Rotating the signing key: generate a new seed, add the new public key to the + binary's trusted key list, then redeploy with the new secret. + +## Observability + +`wrangler.toml` carries the dashboard's observability settings (invocation +logs and traces, persisted, sampled at 100%) so a redeploy keeps them. The +Worker writes no log line of its own; what the platform persists is its +invocation record: route, method, status, timing and request metadata, which +includes the caller's network address and the `X-Codeaf-Install` header for +the platform's retention period. That record is the one place a row can be +tied to an address. A relay that should keep none sets `persist = false` or +`head_sampling_rate = 0` in both blocks. + +## Run your own relay + +Generate your own keypair with the same openssl lines as step 4, then +`wrangler deploy` to your own account with your own route or the workers.dev +address, `wrangler secret put POOL_SIGNING_KEY` to set the seed, and your own +`POOL_PUBLIC_KEY` in your own `wrangler.toml`. Point installs at it with +`CODEAF_MODEL_POOL_RELAY_URL` and `CODEAF_MODEL_POOL_PUBLIC_KEY` (the trusted +key). + +The mirror workflow is optional; it takes the same two values as its workflow +env. `relay/` holds no secret and no value that is not documented in +`wrangler.toml` or this runbook — the quota, the min installs and the cron — so +a copy of the directory is a complete relay. + +## Repository settings, for the optional mirror + +- **Actions:** enable Actions on the repository if it is disabled. +- **Workflow permissions:** under Settings → Actions → General → Workflow + permissions, set "Read and write", which the mirror needs to push the data + branch. +- **Run it once by hand:** the workflow can be dispatched only once its file + is on the default branch, so the first run waits for the branch that carries + it to merge; until then the mirror address answers 404, which the client + treats as one more unreachable mirror and reads its cache or the seed. + Then dispatch the mirror workflow with + `workflow_dispatch`. +- **Confirm it worked:** a commit appears on the `model-pool` branch. diff --git a/docs/design/model-pool/data/measured-crews.csv b/docs/design/model-pool/data/measured-crews.csv new file mode 100644 index 0000000000..75e0f98120 --- /dev/null +++ b/docs/design/model-pool/data/measured-crews.csv @@ -0,0 +1,57 @@ +door,worker,planner,checker,delivered,clean,usd,minutes,runs +task,ds-v4.1-flash,-,qwen3.8-max,0.75,0.69,0.13,0,5 +task,glm-5.3-flash,-,qwen3.8-max,0.95,0.78,0.27,4,17 +task,ds-v4.1-flash,-,kimi-k3,0.75,0.69,0.24,2,7 +task,glm-5.3,-,qwen3.8-max,0.91,0.85,0.53,1,11 +task,ds-v4.1-flash,-,fable-5.1,0.75,0.87,0.45,0,8 +task,glm-5.3-flash,-,kimi-k3,0.95,0.79,0.50,11,21 +do,ds-v4.1-flash,opus-5,-,0.67,0.9,0.46,1,12 +do,ds-v4.1-flash,glm-5.3-flash,-,0.67,0.84,0.36,0,16 +task,ds-v4.1-flash,-,opus-5,0.75,0.64,0.40,0,7 +do,ds-v4.1-flash,qwen3.8-max,-,0.67,0.56,0.22,0,11 +do,ds-v4.1-flash,glm-5.3,-,0.67,0.51,0.23,0,9 +task,muse-spark-1.3,-,qwen3.8-max,0.5,0.95,0.41,0,16 +task,glm-5.3-flash,-,fable-5.1,0.95,0.95,0.95,1,25 +task,glm-5.3,-,kimi-k3,0.91,0.85,0.98,6,14 +task,glm-5.3-flash,-,opus-5,0.95,0.74,0.84,4,20 +do,muse-spark-1.3,qwen3.8-max,-,0.75,0.82,0.72,0,32 +do,ds-v4.1-flash,kimi-k3,-,0.67,0.39,0.18,0,11 +do,muse-spark-1.3,glm-5.3,-,0.75,0.77,0.75,0,26 +do,glm-5.3,qwen3.8-max,-,0.92,0.72,0.93,2,22 +task,muse-spark-1.3,-,kimi-k3,0.5,0.95,0.76,0,19 +do,muse-spark-1.3,kimi-k3,-,0.75,0.65,0.59,0,32 +do,glm-5.3-flash,opus-5,-,0.64,0.95,0.96,1,37 +do,glm-5.3-flash,qwen3.8-max,-,0.64,0.66,0.47,2,34 +do,glm-5.3-flash,glm-5.3,-,0.64,0.6,0.49,1,27 +do,glm-5.3,kimi-k3,-,0.92,0.55,0.76,1,23 +do,ds-v4.1-flash,fable-5.1,-,0.67,0.71,0.89,0,13 +do,muse-spark-1.3,glm-5.3-flash,-,0.75,0.95,1.16,0,48 +do,glm-5.3,glm-5.3-flash,-,0.92,0.95,1.50,1,33 +do,muse-spark-1.3,opus-5,-,0.75,0.95,1.45,0,35 +task,glm-5.3,-,fable-5.1,0.91,0.95,1.87,1,16 +do,glm-5.3-flash,kimi-k3,-,0.64,0.49,0.39,3,34 +do,glm-5.3,opus-5,-,0.92,0.95,1.88,2,24 +task,glm-5.3,-,opus-5,0.91,0.8,1.65,1,13 +task,muse-spark-1.3,-,opus-5,0.5,0.9,1.28,0,19 +task,qwen3.8-max,-,kimi-k3,0.83,0.69,1.31,2,22 +task,muse-spark-1.3,-,fable-5.1,0.5,0.95,1.44,0,24 +do,qwen3.8-max,glm-5.3-flash,-,0.86,0.84,2.01,1,55 +do,glm-5.3-flash,fable-5.1,-,0.64,0.81,1.87,5,41 +task,qwen3.8-max,-,fable-5.1,0.83,0.87,2.50,2,27 +do,qwen3.8-max,glm-5.3,-,0.86,0.5,1.30,1,29 +task,ds-v4-pro-0813,-,qwen3.8-max,0.5,0.35,0.47,0,7 +do,qwen3.8-max,opus-5,-,0.86,0.9,2.53,2,40 +do,muse-spark-1.3,fable-5.1,-,0.75,0.95,2.84,2,39 +do,ds-v4-pro-0813,glm-5.3-flash,-,0.67,0.5,1.32,0,20 +do,ds-v4-pro-0813,opus-5,-,0.67,0.56,1.66,0,14 +do,qwen3.8-max,kimi-k3,-,0.86,0.39,1.02,0,37 +task,qwen3.8-max,-,opus-5,0.83,0.64,2.22,0,22 +task,ds-v4-pro-0813,-,fable-5.1,0.5,0.54,1.64,0,10 +task,ds-v4-pro-0813,-,kimi-k3,0.5,0.36,0.86,0,8 +do,glm-5.3,fable-5.1,-,0.92,0.87,3.68,4,27 +do,ds-v4-pro-0813,qwen3.8-max,-,0.67,0.23,0.82,0,13 +task,ds-v4-pro-0813,-,opus-5,0.5,0.3,1.46,0,8 +do,ds-v4-pro-0813,glm-5.3,-,0.67,0.17,0.85,0,11 +do,qwen3.8-max,fable-5.1,-,0.86,0.71,4.94,1,44 +do,ds-v4-pro-0813,fable-5.1,-,0.67,0.38,3.24,1,16 +do,ds-v4-pro-0813,kimi-k3,-,0.67,0.06,0.67,0,13 diff --git a/docs/design/model-pool/data/seed-cells.csv b/docs/design/model-pool/data/seed-cells.csv new file mode 100644 index 0000000000..90285f63e3 --- /dev/null +++ b/docs/design/model-pool/data/seed-cells.csv @@ -0,0 +1,16 @@ +role,model,mean,sd,n +high,anthropic/claude-fable-5.1,87.1,5.9,29 +high,anthropic/claude-opus-5,83.3,14.1,4 +high,moonshotai/kimi-k3,86.2,6.3,27 +high,qwen/qwen3.8-max-0902,92.0,5.3,23 +mastermind,anthropic/claude-fable-5.1,77.5,6.5,38 +mastermind,anthropic/claude-opus-5,81.0,8.4,19 +mastermind,moonshotai/kimi-k3,57.1,10.6,19 +mastermind,qwen/qwen3.8-max-0902,50.0,16.7,6 +mastermind,z-ai/glm-5.3,62.5,16.1,6 +worker,deepseek/deepseek-v4.1-flash,83.3,6.1,34 +worker,meta/muse-spark-1.3,94.1,5.5,15 +worker,openai/gpt-5.6-sol,85.7,12.4,5 +worker,qwen/qwen3.8-max-0902,72.7,12.9,9 +worker,z-ai/glm-5.3,72.5,7.0,38 +worker,z-ai/glm-5.3-flash,67.4,4.9,90 diff --git a/docs/design/model-pool/fig-cells.pdf b/docs/design/model-pool/fig-cells.pdf new file mode 100644 index 0000000000..4ba5204fe9 Binary files /dev/null and b/docs/design/model-pool/fig-cells.pdf differ diff --git a/docs/design/model-pool/fig-front.pdf b/docs/design/model-pool/fig-front.pdf new file mode 100644 index 0000000000..e6931cdfef Binary files /dev/null and b/docs/design/model-pool/fig-front.pdf differ diff --git a/docs/design/model-pool/fig-regret.pdf b/docs/design/model-pool/fig-regret.pdf new file mode 100644 index 0000000000..16c0897db2 Binary files /dev/null and b/docs/design/model-pool/fig-regret.pdf differ diff --git a/docs/design/model-pool/fig-semibandit.pdf b/docs/design/model-pool/fig-semibandit.pdf new file mode 100644 index 0000000000..5cda759ece Binary files /dev/null and b/docs/design/model-pool/fig-semibandit.pdf differ diff --git a/docs/design/model-pool/fig-shrink.pdf b/docs/design/model-pool/fig-shrink.pdf new file mode 100644 index 0000000000..981e0c5a5b Binary files /dev/null and b/docs/design/model-pool/fig-shrink.pdf differ diff --git a/docs/design/model-pool/figures.py b/docs/design/model-pool/figures.py new file mode 100644 index 0000000000..d33f7b1a2e --- /dev/null +++ b/docs/design/model-pool/figures.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Draws the figures in pareto-crewing.tex from data/*.csv. Standard library, numpy and matplotlib only. + + python3 figures.py # writes fig-*.pdf and tab-*.tex beside this file + +The simulations are the ones the paper describes. Ground truth is the table of measured crews fitted +from our own scored runs (data/measured-crews.csv). A learner picks a crew per run, sees the run's +outcome and its dollars, and pays the all-in cost E = c + lambda*t + H*(1 - clean). Regret is E minus +the oracle crew's, averaged over replicates. Every replicate draws its own noisy catalog prior. +""" +import csv, os, random +import numpy as np +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +HERE = os.path.dirname(os.path.abspath(__file__)) +TABLE_PROVENANCE = "% generated by figures.py from data/measured-crews.csv; do not edit\n" +H, LAM, K = 10.0, 0.01, 30 +REPS = 200 +plt.rcParams.update({"font.size": 9, "font.family": "serif", "axes.spines.top": False, "axes.spines.right": False}) + + +def crews(): + rows = list(csv.DictReader(open(os.path.join(HERE, "data", "measured-crews.csv")))) + for r in rows: + for k in ("delivered", "clean", "usd", "minutes"): + r[k] = float(r[k]) + r["runs"] = int(r["runs"]) + r["name"] = r["worker"] + ("+" + r["checker"] if r["checker"] != "-" else "") + ("/" + r["planner"] if r["planner"] != "-" else "") + r["run"] = r["usd"] + LAM * r["minutes"] + r["p"] = r["delivered"] * r["clean"] + return rows + + +def allin(r, h=H): + return r["run"] + h * (1 - r["p"]) + + +def fig_front(): + rows = crews() + fig, ax = plt.subplots(figsize=(5.4, 3.5)) + for door, mark, label in (("task", "o", "chat door (worker + checker)"), ("do", "s", "headless door (worker + planner)")): + pts = [r for r in rows if r["door"] == door] + ax.scatter([r["run"] for r in pts], [100 * r["p"] for r in pts], s=18, marker=mark, alpha=.75, label=label) + front, best = [], -1 + for r in sorted(rows, key=lambda r: r["run"]): + if r["p"] > best: + front.append(r) + best = r["p"] + ax.plot([r["run"] for r in front], [100 * r["p"] for r in front], "k-", lw=.8, label="pareto front") + for r in front: + ax.annotate(r["name"], (r["run"], 100 * r["p"]), fontsize=6, xytext=(4, -9), textcoords="offset points") + xs = np.linspace(0.05, 4, 100) + for h, e, x in ((1, 1.2, 1.0), (10, 3.0, 2.4), (50, 12.0, 3.2)): + ax.plot(xs, 100 * (1 - (e - xs) / h), color="#999", lw=.6, ls="--") + ax.text(x, 100 * (1 - (e - x) / h) + 1.5, f"E = \\${e:.0f} at H = \\${h}", fontsize=6, color="#777") + ax.set_xlim(0, 4) + ax.set_ylim(0, 100) + ax.set_xlabel("cost of one run, $ (dollars + λ · minutes)") + ax.set_ylabel("P(delivered and clean), %") + ax.legend(fontsize=7, loc="lower left") + fig.tight_layout() + fig.savefig(os.path.join(HERE, "fig-front.pdf")) + + +def fig_cells(): + rows = list(csv.DictReader(open(os.path.join(HERE, "data", "seed-cells.csv")))) + fig, axes = plt.subplots(1, 3, figsize=(6.2, 2.6)) + for ax, role in zip(axes, ("worker", "high", "mastermind")): + cells = sorted([r for r in rows if r["role"] == role], key=lambda r: -float(r["mean"])) + names = [r["model"].split("/")[-1] for r in cells] + ax.barh(names, [float(r["mean"]) for r in cells], xerr=[float(r["sd"]) for r in cells], color="#8a8fb0", ecolor="#333", capsize=2) + for i, r in enumerate(cells): + ax.text(2, i, f"n={r['n']}", va="center", fontsize=6, color="white") + ax.set_title(role, fontsize=9) + ax.set_xlim(0, 100) + ax.invert_yaxis() + ax.tick_params(axis="y", labelsize=7) + axes[0].set_xlabel("role_quality, posterior mean ± sd") + fig.tight_layout() + fig.savefig(os.path.join(HERE, "fig-cells.pdf")) + + +def fig_shrink(): + q_cat, s_bar, sigma = 80.0, 65.0, 15.0 + n = np.arange(0, 121) + mean = (K * q_cat + n * s_bar) / (K + n) + sd = sigma / np.sqrt(K + n) + fig, ax = plt.subplots(figsize=(5.2, 2.8)) + ax.fill_between(n, mean - 1.96 * sd, mean + 1.96 * sd, color="#8a8fb0", alpha=.3, label="95% posterior band") + ax.plot(n, mean, "k-", label="posterior mean") + ax.axhline(q_cat, ls=":", color="#555", lw=.8) + ax.axhline(s_bar, ls="--", color="#555", lw=.8) + ax.text(2, q_cat + 0.8, "catalog prior", fontsize=7) + ax.text(100, s_bar + 0.8, "measured mean", fontsize=7) + ax.axvline(K, color="#a55", lw=.8) + ax.text(K + 1.5, 62, "n = k = 30: half weight each", fontsize=7, color="#a55") + ax.set_xlabel("effective observations n") + ax.set_ylabel("role quality") + ax.set_xlim(0, 120) + ax.set_ylim(60, 84) + ax.legend(fontsize=7, loc="upper right") + fig.tight_layout() + fig.savefig(os.path.join(HERE, "fig-shrink.pdf")) + + +def simulate(strategy, N, seed, rows, oracle): + """Crew-level learners: every crew is one arm with one Beta posterior on P(clean).""" + rnd = random.Random(seed) + truth = {r["name"]: r["p"] for r in rows} + prior = {n: min(.99, max(.01, p + rnd.gauss(0, .15))) for n, p in truth.items()} + a = {n: 1 + K * prior[n] for n in truth} + b = {n: 1 + K * (1 - prior[n]) for n in truth} + fixed = {r["name"]: r["run"] for r in rows} + total, explored = 0.0, 0 + for t in range(N): + mean_best = min(truth, key=lambda n: fixed[n] + H * (1 - a[n] / (a[n] + b[n]))) + if strategy == "table": + pick = rows[1]["name"] + elif strategy == "catalog": + pick = min(truth, key=lambda n: fixed[n] + H * (1 - prior[n])) + elif strategy == "greedy": + pick = mean_best if rnd.random() >= 0.1 else rnd.choice(list(truth)) + else: + pick = min(truth, key=lambda n: fixed[n] + H * (1 - rnd.betavariate(a[n], b[n]))) + explored += pick != mean_best + clean = rnd.random() < truth[pick] + total += fixed[pick] + H * (1 - clean) - oracle + if strategy in ("greedy", "thompson"): + a[pick] += clean + b[pick] += not clean + return total / N, explored / N + + +def fig_regret(): + rows = [r for r in crews() if r["door"] == "task"] + oracle = min(allin(r) for r in rows) + Ns = [10, 20, 40, 70, 100, 150, 200, 300, 400] + fig, (ax, ax2) = plt.subplots(1, 2, figsize=(6.2, 2.8)) + table = {} + for s, style in (("table", "k:"), ("catalog", "k--"), ("greedy", "C1-"), ("thompson", "C0-")): + reg, exp = [], [] + for N in Ns: + rs = [simulate(s, N, seed, rows, oracle) for seed in range(REPS)] + reg.append(np.mean([r[0] for r in rs])) + exp.append(np.mean([r[1] for r in rs])) + table[s] = reg + ax.plot(Ns, reg, style, label={"table": "fixed table row", "catalog": "catalog only", "greedy": "ε-greedy, ε = 0.1", "thompson": "Thompson"}[s]) + if s in ("greedy", "thompson"): + ax2.plot(Ns, [100 * e for e in exp], style, label=s) + ax.set_xlabel("runs N") + ax.set_ylabel("regret vs oracle, $ per task") + ax.set_ylim(0, None) + ax.legend(fontsize=7) + ax2.set_xlabel("runs N") + ax2.set_ylabel("runs off the posterior-mean pick, %") + ax2.set_ylim(0, None) + ax2.legend(fontsize=7) + fig.tight_layout() + fig.savefig(os.path.join(HERE, "fig-regret.pdf")) + with open(os.path.join(HERE, "tab-regret.tex"), "w") as f: + f.write(TABLE_PROVENANCE) + f.write("\\begin{tabular}{l" + "r" * len(Ns) + "}\\toprule\n$N$ & " + " & ".join(str(n) for n in Ns) + "\\\\\\midrule\n") + for s in ("table", "catalog", "greedy", "thompson"): + f.write(s + " & " + " & ".join(f"{v:.2f}" for v in table[s]) + "\\\\\n") + f.write("\\bottomrule\\end{tabular}\n") + + +VENDOR = {"ds-v4.1-flash": "deepseek", "ds-v4-pro-0813": "deepseek", "ds-v4-flash-0731": "deepseek", "glm-5.3-flash": "z-ai", + "glm-5.3": "z-ai", "muse-spark-1.3": "meta", "qwen3.8-max": "qwen", "kimi-k3": "moonshot", "fable-5.1": "anthropic", + "opus-5": "anthropic"} + + +def role_truth(): + """A three-role ground truth read off the measured rows under the linear rule of the paper: a role quality + q_r(m) in [0, 100] per (role, model) and a dollar surcharge per (role, model); a crew's P(clean) is the mean of + its role qualities over 100 and its cost the sum of its surcharges. Workers and checkers come from the chat + rows (delivered is the worker's, clean the checker's), planners from the headless rows.""" + rows = crews() + chat = [r for r in rows if r["door"] == "task"] + do = [r for r in rows if r["door"] == "do"] + mean = lambda xs: float(np.mean(xs)) + q = {"worker": {}, "checker": {}, "planner": {}} + cost = {"worker": {}, "checker": {}, "planner": {}} + for w in sorted({r["worker"] for r in chat}): + mine = [r for r in chat if r["worker"] == w] + q["worker"][w] = 100 * mean([r["delivered"] for r in mine]) + cost["worker"][w] = mean([r["run"] for r in mine]) + base = mean([r["run"] for r in chat]) + for c in sorted({r["checker"] for r in chat}): + mine = [r for r in chat if r["checker"] == c] + q["checker"][c] = 100 * mean([r["clean"] for r in mine]) + cost["checker"][c] = mean([r["run"] for r in mine]) - base + dobase = mean([r["run"] for r in do]) + for p in sorted({r["planner"] for r in do}): + mine = [r for r in do if r["planner"] == p] + q["planner"][p] = 100 * mean([r["clean"] for r in mine]) + cost["planner"][p] = max(0.0, mean([r["run"] for r in mine]) - dobase) + grid = {} + for w in q["worker"]: + for c in q["checker"]: + if VENDOR.get(w) == VENDOR.get(c): + continue + for p in q["planner"]: + k = (w, c, p) + grid[k] = dict(p=(q["worker"][w] + q["checker"][c] + q["planner"][p]) / 300, + run=cost["worker"][w] + cost["checker"][c] + cost["planner"][p]) + return q, grid + + +def simulate_roles(level, N, seed, q, grid, sigma=15.0): + """Thompson sampling under three kinds of feedback on the same truth: one Beta arm per crew fed the run's + clean/defect bit; one Gaussian arm per crew fed the mean of the judge's role scores; one Gaussian posterior per + (role, model) fed each role's own judge score (semi-bandit). Judge scores are the role quality plus N(0, sigma).""" + rnd = random.Random(seed) + arms = list(grid) + roles = ("worker", "checker", "planner") + oracle = min(grid[a]["run"] + H * (1 - grid[a]["p"]) for a in arms) + noise = lambda x: min(99.0, max(1.0, x + rnd.gauss(0, 15))) + prior_q = {r: {m: noise(q[r][m]) for m in q[r]} for r in roles} + prior_p = {a: sum(prior_q[r][a[i]] for i, r in enumerate(roles)) / 300 for a in arms} + if level == "crew-bit": + a_ = {a: 1 + K * prior_p[a] for a in arms} + b_ = {a: 1 + K * (1 - prior_p[a]) for a in arms} + elif level == "crew-score": + n_ = {a: float(K) for a in arms} + s_ = {a: K * 100 * prior_p[a] for a in arms} + else: + n_ = {(r, m): float(K) for r in roles for m in q[r]} + s_ = {(r, m): K * prior_q[r][m] for r in roles for m in q[r]} + total = 0.0 + for t in range(N): + if level == "crew-bit": + pick = min(arms, key=lambda a: grid[a]["run"] + H * (1 - rnd.betavariate(a_[a], b_[a]))) + elif level == "crew-score": + pick = min(arms, key=lambda a: grid[a]["run"] + H * (1 - rnd.gauss(s_[a] / n_[a], sigma / n_[a] ** .5) / 100)) + else: + draw = {k: rnd.gauss(s_[k] / n_[k], sigma / n_[k] ** .5) for k in n_} + pick = min(arms, key=lambda a: grid[a]["run"] + H * (1 - sum(draw[(r, a[i])] for i, r in enumerate(roles)) / 300)) + g = grid[pick] + clean = rnd.random() < g["p"] + total += g["run"] + H * (1 - clean) - oracle + scores = [q[r][pick[i]] + rnd.gauss(0, sigma) for i, r in enumerate(roles)] + if level == "crew-bit": + a_[pick] += clean; b_[pick] += not clean + elif level == "crew-score": + n_[pick] += 1; s_[pick] += float(np.mean(scores)) + else: + for i, r in enumerate(roles): + n_[(r, pick[i])] += 1; s_[(r, pick[i])] += scores[i] + return total / N + + +def fig_semibandit(): + q, grid = role_truth() + Ns = [10, 20, 40, 70, 100, 150, 200, 300, 400] + fig, ax = plt.subplots(figsize=(4.8, 2.9)) + table = {} + for level, style, label in (("crew-bit", "C0:", "one arm per crew, clean/defect bit"), + ("crew-score", "C0--", "one arm per crew, mean judge score"), + ("role", "C2-", "one posterior per (role, model), role scores")): + reg = [np.mean([simulate_roles(level, N, seed, q, grid) for seed in range(REPS)]) for N in Ns] + table[level] = reg + ax.plot(Ns, reg, style, label=label) + d = sum(len(q[r]) for r in q) + ax.set_title(f"{len(grid)} crews from {len(q['worker'])} workers × {len(q['checker'])} checkers × {len(q['planner'])} planners; {d} unknowns", fontsize=8) + ax.set_xlabel("runs N") + ax.set_ylabel("regret vs oracle, $ per task") + ax.set_ylim(0, None) + ax.legend(fontsize=7) + fig.tight_layout() + fig.savefig(os.path.join(HERE, "fig-semibandit.pdf")) + with open(os.path.join(HERE, "tab-semibandit.tex"), "w") as f: + f.write(TABLE_PROVENANCE) + f.write("\\begin{tabular}{l" + "r" * len(Ns) + "}\\toprule\n$N$ & " + " & ".join(str(n) for n in Ns) + "\\\\\\midrule\n") + for level, name in (("crew-bit", "crew, clean bit"), ("crew-score", "crew, mean score"), ("role", "role, semi-bandit")): + f.write(name + " & " + " & ".join(f"{v:.2f}" for v in table[level]) + "\\\\\n") + f.write("\\bottomrule\\end{tabular}\n") + + +if __name__ == "__main__": + fig_front() + fig_cells() + fig_shrink() + fig_regret() + fig_semibandit() + print("figures written") diff --git a/docs/design/model-pool/pareto-crewing.pdf b/docs/design/model-pool/pareto-crewing.pdf new file mode 100644 index 0000000000..7771ef6318 Binary files /dev/null and b/docs/design/model-pool/pareto-crewing.pdf differ diff --git a/docs/design/model-pool/pareto-crewing.tex b/docs/design/model-pool/pareto-crewing.tex new file mode 100644 index 0000000000..c57a7ac586 --- /dev/null +++ b/docs/design/model-pool/pareto-crewing.tex @@ -0,0 +1,567 @@ +\documentclass[10pt,a4paper]{article} +\usepackage[margin=2.54cm]{geometry} +\usepackage{amsmath,amssymb,amsthm,mathtools} +\usepackage{newtxtext,newtxmath} +\usepackage{booktabs} +\usepackage{graphicx} +\usepackage{microtype} +\usepackage{authblk} +\usepackage{algorithm} +\usepackage{algpseudocode} +\usepackage{tikz} +\usetikzlibrary{arrows.meta,positioning,shapes.geometric,calc,fit} +\usepackage{fancyhdr} +\usepackage[hidelinks]{hyperref} +\usepackage{caption} +\usepackage{float} +\captionsetup{font=small,labelfont=bf,width=.92\textwidth} +\usepackage{abstract} +\renewcommand{\abstractnamefont}{\normalfont\bfseries} +\setlength{\absleftindent}{1.2cm}\setlength{\absrightindent}{1.2cm} +\setlength{\parskip}{2pt} + +\pagestyle{fancy} +\fancyhf{} +\fancyhead[L]{\small\textsc{Preprint}} +\fancyhead[R]{\small Pareto Crewing} +\fancyfoot[C]{\small\thepage} +\renewcommand{\headrulewidth}{0.2pt} + +\newcommand{\E}{\mathbb{E}} +\newcommand{\R}{\mathbb{R}} +\newcommand{\N}{\mathbb{N}} +\newcommand{\roles}{\mathcal{R}} +\newcommand{\mods}{\mathcal{M}} +\newcommand{\crews}{\mathcal{K}} +\newcommand{\front}{\mathcal{F}} +\newcommand{\seat}[1]{\mathrm{#1}} +\newcommand{\code}[1]{\texttt{#1}} +\newcommand{\pclean}{P_{\mathrm{c}}} +\DeclareMathOperator*{\argmin}{arg\,min} +\DeclareMathOperator*{\argmax}{arg\,max} +\newtheorem{theorem}{Theorem} +\newtheorem{proposition}{Proposition} +\newtheorem{lemma}{Lemma} +\newtheorem{corollary}{Corollary} +\theoremstyle{definition} +\newtheorem{definition}{Definition} +\newtheorem{assumption}{Assumption} +\theoremstyle{remark} +\newtheorem{remark}{Remark} + +\title{\textbf{Pareto Crewing}\\[6pt] +\large Decentralised, privacy-preserving measurement of language-model teams,\\ and a learner that picks them on the cost-quality front, per role and per task} +\author{Santosh Kumar Radha} +\affil{AgentField AI} +\date{September 2026} + +\begin{document} +\maketitle +\thispagestyle{fancy} + +\begin{abstract} +\noindent +\emph{Pareto crewing} is the practice of picking the language-model team for a task on the cost-quality front, one model per role, from evidence about how models have performed in that role rather than from a fixed table. An agent harness runs a task through a \emph{crew}: an assignment of one language model to each of $n$ roles (in our instance a worker that does the work, a checker that reads it and a planner that cuts it). The crew that pays best is neither the cheapest nor the strongest on paper; it is the one that minimises the expected cost of one clean delivery once failed runs, wall time and shipped defects are priced. We give that objective in closed form, show how a crew is computed from a published catalog of prices and benchmark indexes alone (a pareto front in bill and quality, with a scale-free knee), and then how measurements from real runs replace the catalog's proxy. Runs are scored role by role by a model outside the crew. Each score becomes a row that carries no text, no file name, no price and no identity beyond a per-install nonce, so what an install sends is a handful of coarse fields about a model, not about the work; rows reduce to sufficient statistics that form a join-semilattice, so any number of installs and a relay can merge them in any order and reach the same state, the relay is a convenience rather than an authority, and the published index is signed so that every reader can check what it is blending in. Participation is a setting: an install can contribute, read and learn from its own runs without sending, or stay out. Judge severity is removed by a two-way fixed-effects fit on those statistics, drift by a local-level model whose forgetting rate is estimated rather than set, and heterogeneity between installs by a hierarchical prior whose variance is estimated by moments; every statistical constant is an empirical-Bayes estimate, and what remains for the user to set is the price of a defect and the price of time. The crew is chosen by Thompson sampling with semi-bandit feedback: because the objective is linear in role qualities and every role is scored separately, a run on one crew informs every crew that shares a model with it, and the sample complexity scales with the number of models rather than the number of crews. On a ground truth fitted to our own runs the learner beats a fixed table by a third of the remaining regret within a few hundred runs, and role-level feedback halves the regret of a crew-level learner fed the same scores at four hundred runs. +\end{abstract} + +\vspace{4pt} +\noindent\textbf{Keywords:} Pareto crewing, model selection, pareto front, empirical Bayes, Thompson sampling, combinatorial semi-bandits, CRDT, judge calibration, privacy, decentralised measurement. + +\section{Introduction} + +A harness that drives language models through software tasks has to decide, before every task, which model sits in which role. The decision is made under three kinds of information that arrive at different times and in different units: published prices and benchmark indexes, available for every model on the day it appears; the harness's own past runs, few but on the right distribution; and the runs of everyone else using the same harness, many but on other people's tasks and scored by other people's judges. This note is about turning the three into one decision. + +We call the resulting discipline \emph{Pareto crewing}: the crew is picked on the cost-quality front, per role and per task, from whatever evidence exists on the day, and the evidence is pooled across installs without any install giving up what it worked on. Our contributions are the following. +\begin{enumerate} +\item A closed-form objective for a crew, the expected cost of one clean delivery under reruns and a priced defect (Section~\ref{sec:objective}), and the observation that on a catalog alone the objective cannot be evaluated, which is why the catalog picker is a pareto front with a scale-free knee (Section~\ref{sec:catalog}). +\item A decentralised measurement of role quality (Section~\ref{sec:pool}): text-free rows, sufficient statistics that form a state-based CRDT (Proposition~\ref{prop:crdt}), judge fixed effects identified from the connectivity of the judge-cell graph (Proposition~\ref{prop:ident}), robust aggregation across installs, and drift handled by a local-level model. +\item An estimator with no tuned constant except the user's price of a defect (Section~\ref{sec:eb}): the catalog's prior weight, the between-install variance, the noise scale and the forgetting rate are all estimated from the pool by moments. +\item A picker (Section~\ref{sec:decide}) that is Thompson sampling in a combinatorial semi-bandit: the crew objective is linear in role qualities, roles are scored separately, so one run updates every crew sharing a model with it. +\item An empirical check on our own measured runs (Section~\ref{sec:exp}), reproducible from the script beside this document. +\end{enumerate} + +\section{Setting} +\label{sec:setting} + +\begin{definition}[Roles, crews] +Let $\roles$ be a finite set of $n$ \emph{roles} and $\mods$ a finite set of models. A \emph{crew} is a map $\kappa:\roles\to\mods$. Each role $r$ carries a \emph{shape}: a token share $v_r\ge 0$ with $\sum_r v_r=1$, index weights $w_r\in\Delta^{2}$, a cache share $\gamma_r\in[0,1]$, an input-to-output ratio $\rho_r>0$, and a predicate $\mathrm{fit}_r(m)$ saying whether model $m$ can hold the role (tools, images, context). A symmetric relation $D\subseteq\roles\times\roles$ names the pairs of roles that may not be held by models of the same vendor. +\end{definition} + +Our instance has $\roles=\{\seat{W},\seat{H},\seat{M}\}$ (worker, high, mastermind), token shares $(0.90,0.08,0.02)$, weights $(0.2,0.3,0.5)$, $(\tfrac13,\tfrac13,\tfrac13)$, $(0.6,0,0.4)$ on the indexes (intelligence, coding, agentic), cache shares $(0.75,0.75,0)$, ratios $(100,100,10)$ and $D=\{(\seat W,\seat H)\}$: the two roles that read the same work never share one shop. On the chat door the planner role is absent and $n=2$. Nothing below depends on $n=3$. + +A \emph{task} is one unit of work; a \emph{run} is one attempt at it by a crew. A run ends \emph{delivered} or not, and a delivered task is later found \emph{clean} (no major defect) or not. Money is in dollars and time is priced at $\lambda$ dollars per minute. + +The \emph{catalog} lists, per model, a prompt price $\pi_m$, a cache-read price $\pi^c_m$, a completion price $\kappa_m$ per million tokens, and benchmark indexes $I_m\in\R^3$. The \emph{pool} is the set of measurements described in Section~\ref{sec:pool}; the \emph{own sheet} is one install's part of it. + +\section{The crew computed from the catalog} +\label{sec:catalog} + +\paragraph{Role cost and role quality.} +Per million task tokens on role $r$, model $m$ costs +\begin{equation} +c_r(m) = (1-\gamma_r)\,\pi_m + \gamma_r\,\pi^{c}_m + \kappa_m/\rho_r , +\label{eq:seatcost} +\end{equation} +and its catalog quality is the weighted, pool-normalised index +\begin{equation} +\hat q_r(m) = 100 \sum_{i=1}^{3} w_{r,i}\,\frac{I_{m,i}}{\max_{m'\in\mods} I_{m',i}} . +\label{eq:catq} +\end{equation} +A missing index is imputed from the models that publish it: for each index the model does publish, the median over at least three donor models of the ratio between the missing and the present index gives an offer, the offers are averaged, and the estimate is capped at the largest measured value of the missing index in the pool; with fewer than three donors the model takes the standing its present indexes give it on the pool's scale. An imputed quality is flagged and loses ties. + +\paragraph{Shortlists.} +Role $r$'s shortlist is +\begin{equation} +S_r=\{m:\ \mathrm{fit}_r(m)\ \wedge\ \hat q_r(m)\ge \phi\max_{m'} \hat q_r(m')\},\qquad \phi=0.8, +\label{eq:floor} +\end{equation} +the floor being the statement that a model which cannot do the job is not cheap at any price. The crew set is $\crews=\{\kappa\in\prod_r S_r:\ \mathrm{vendor}(\kappa(r))\ne\mathrm{vendor}(\kappa(r'))\ \forall (r,r')\in D\}$. + +\paragraph{Bill, quality, front.} +\begin{equation} +B(\kappa)=\sum_{r} v_r\,c_r(\kappa(r)),\qquad Q(\kappa)=\sum_{r}\nu_r\, q_r(\kappa(r)),\qquad \nu_r\ge0,\ \textstyle\sum_r\nu_r=1, +\label{eq:crew} +\end{equation} +with $\nu$ uniform by default. The \emph{front} $\front\subseteq\crews$ is the set of crews not dominated in $(B,Q)$. Sorting $\crews$ by $(B\uparrow, Q\downarrow, \text{imputed count}\uparrow, \text{id}\uparrow)$ and keeping a crew when its $Q$ exceeds every kept crew's yields $\front$ in $O(|\crews|\log|\crews|)$, and the tie order makes $\front$ a function of the set $\crews$ rather than of any input order. In practice $|S_r|$ is a few dozen and $|\crews|$ a few thousand. + +\paragraph{Why a front and not a score.} +The objective of Section~\ref{sec:objective} needs a probability of clean delivery. The catalog has none, so any scalarisation of $(B,Q)$ would fix an exchange rate between a price and a benchmark index that nobody has measured. Dominance is the only order that needs no such rate; every crew a user could sensibly want is on $\front$, which is small, and the remaining choice is one-dimensional along it. + +\paragraph{Knee and knob.} +Index $\front$ by bill, $B_0\le\dots\le B_1$. The three named picks are \emph{frugal} $=\argmin_\front B$, \emph{max} $=\argmax_\front Q$, and \emph{balanced}, the crew farthest above the chord of the front in $(\ln B,Q)$: +\begin{equation} +\text{balanced}=\argmax_{\kappa\in\front}\ \Big[(Q(\kappa)-Q_0) - (Q_1-Q_0)\frac{\ln B(\kappa)-\ln B_0}{\ln B_1-\ln B_0}\Big]. +\label{eq:knee} +\end{equation} +\begin{proposition}[Scale invariance] +The balanced crew is invariant under $B\mapsto aB$ for any $a>0$ and under $Q\mapsto Q+b$. +\end{proposition} +\begin{proof} +$\ln(aB)-\ln(aB_0)=\ln B-\ln B_0$ and $\ln(aB_1)-\ln(aB_0)=\ln B_1-\ln B_0$; the bracket is unchanged. A shift of $Q$ cancels in both differences. +\end{proof} +The log scale is not cosmetic: catalog prices span two orders of magnitude, and the knee should not move when a currency changes or every vendor cuts prices by the same factor. A knob $k\in[0,1]$ picks the best crew with $B\le B_0(B_1/B_0)^k$, so equal steps in $k$ are equal ratios in the bill. + +\section{The objective: the cost of one clean delivery} +\label{sec:objective} + +Let a run of crew $\kappa$ cost $c$ dollars and $t$ minutes, deliver with probability $r$, and let a delivered task be clean with probability $p$; a run that fails to deliver costs $c_f$ and $t_f$. + +\begin{lemma}[Rerun until clean] +If every defect is caught and the task rerun, the expected cost to one clean delivery is +\begin{equation} +E_0(\kappa)=\frac{(1-r)(c_f+\lambda t_f)+r\,(c+\lambda t)}{r\,p}. +\label{eq:E0} +\end{equation} +\end{lemma} +\begin{proof} +Attempts are i.i.d.\ with success probability $rp$; the count to the first success is geometric with mean $1/(rp)$, and by Wald's identity the expected total cost is the mean count times the expected cost of one attempt, which is the numerator. +\end{proof} + +\begin{lemma}[Priced defect] +If a shipped defect costs $H$ to find and mend and only non-deliveries are rerun, +\begin{equation} +E_H(\kappa)=\frac{(1-r)(c_f+\lambda t_f)}{r} + (c+\lambda t) + (1-p)\,H . +\label{eq:EH} +\end{equation} +\end{lemma} +\begin{proof} +The count of failed attempts before the first delivery is geometric on $\{0,1,\dots\}$ with mean $(1-r)/r$; the delivering attempt costs $c+\lambda t$; the defect is paid once with probability $1-p$. +\end{proof} + +$H$ is the one quantity that belongs to the user, and the budget word sets it: frugal $H=\$1$, balanced $\$10$, max $\$50$. Both forms are monotone in the same three quantities, the run's cost, its time and $\pclean=rp$, and the picker works with +\begin{equation} +E(\kappa)=c(\kappa)+\lambda\,t(\kappa)+H\,\big(1-\pclean(\kappa)\big). +\label{eq:E} +\end{equation} +Equation~\eqref{eq:E} is~\eqref{eq:EH} with $r$ set to one: a run that fails to deliver is priced as a shipped defect rather than rerun. The surrogate ranks crews as $E_H$ does when $H$ is large next to a run's cost and $r$ is not far below one, which is the regime of every crew on the front of Figure~\ref{fig:front}; when a role's delivered rate is measured, as the worker's is, the picker can use~\eqref{eq:EH} directly and nothing below changes. Figure~\ref{fig:front} draws our measured crews with the iso-lines of~\eqref{eq:E}. The knob $k$ and the price $H$ parametrise the same trade-off; $k$ is what a catalog admits, $H$ is what measurements admit. + +\begin{assumption}[Linearity in roles] +\label{ass:linear} +$\pclean(\kappa)=\sum_r \nu_r\,q_r(\kappa(r))/100$, where $q_r(m)$ is the pool's role quality: the run's outcome credited to the role, on a 0--100 scale. +\end{assumption} +The assumption is what makes~\eqref{eq:E} linear in the role qualities, which Section~\ref{sec:decide} uses. It is a working rule rather than a law: a product over roles would count one outcome $n$ times, and a fitted interaction becomes available once judged scores separate the roles' contributions. Nothing in Sections~\ref{sec:pool}--\ref{sec:eb} depends on it. + +\begin{figure}[t] +\centering\includegraphics{fig-front.pdf} +\caption{Our measured crews: 56 (worker, checker or planner) combinations, each fitted by a ridge model on 72 scored runs over nine tasks; cost of one run against the probability that it delivers clean. Solid: the pareto front. Dashed: iso-cost lines of~\eqref{eq:E}; a larger $H$ tilts them so that quality buys more.} +\label{fig:front} +\end{figure} + +\section{The Model Pool: a decentralised measure of role quality} +\label{sec:pool} + +\subsection{Judged runs and rows} + +After a run finishes, one model outside the crew reads the record (brief, deliverable, the checker's report and claim, the files touched) and answers one question per role with a score in $[0,100]$ and one sentence. The judge is chosen deterministically: the cheapest catalog model with coding index at least $60$ that can call tools, holds no role in the crew, and is not from the worker's vendor; a role the judge's own model holds is skipped. The floor is absolute, not relative to the worker; its reason is that a judge well below the class of the work it reads rubber-stamps. The judge's calls are billed to a role of their own, so learning is never free by accident; the cost is one short completion per role. + +One judged role of one run is one \emph{row} +\[ +x=(\text{role},\ \text{model},\ \text{score},\ \text{judge},\ \text{door},\ \text{size}\in\{S,M,L\},\ \text{day}), +\] +sent under an \emph{install} nonce drawn once per profile and carried on the batch, not in the row. No text, no price and no duration is in a row; dollars and minutes stay in the install's own ledger. Rows wait in a local outbox, which can be inspected, until they are sent. + +\subsection{Sufficient statistics and order-free merging} + +\begin{definition}[Cell, sheet] +A \emph{cell} is a tuple $\theta=(\text{role},\text{model},\text{judge},\text{door},\text{size})$. A \emph{sheet} is a map from (install, day, cell) to a triple $T=(n,S,S^2)\in\N\times\R_{\ge0}\times\R_{\ge0}$, the count, sum and sum of squares of the scores an install has produced in that cell. +\end{definition} + +Every quantity computed below is a function of the sheet. Sheets are merged by the join +\begin{equation} +(A\sqcup B)(u,d,\theta)=\big(\max(n_A,n_B),\ \max(S_A,S_B),\ \max(S^2_A,S^2_B)\big)\quad\text{componentwise on } A(u,d,\theta),\,B(u,d,\theta), +\label{eq:join} +\end{equation} +with a missing entry read as $(0,0,0)$. + +\begin{proposition}[The sheet is a state-based CRDT] +\label{prop:crdt} +Let each install $u$ only ever publish its own entries $(u,d,\theta)$ and let every published triple be the running total of $u$'s rows in $\theta$ on day $d$. Then (i) $\sqcup$ is commutative, associative and idempotent; (ii) an install's triples are monotone non-decreasing in time, so a later publication dominates an earlier one under $\sqcup$; (iii) any two nodes that have received the same set of publications, in any order and with any duplication, hold the same sheet. +\end{proposition} +\begin{proof} +(i) is the componentwise max on a product of totally ordered sets. (ii) Scores lie in $[0,100]$, so $n$, $S$ and $S^2$ each grow by a non-negative amount per row. (iii) follows from (i) and (ii): the join of a set of publications is their least upper bound, which does not depend on order or multiplicity, and by (ii) the least upper bound of an install's publications is its latest. +\end{proof} + +The cell means and pooled variances are functions of $\sum_u T(u,\theta)$, so by Proposition~\ref{prop:crdt} a relay and every install compute the same estimates from the same publications; there is no coordinator whose order of arrival matters, and a relay that is lost is rebuilt from any node that kept the publications. This is the sense in which the measure is decentralised: the relay is a convenience for bandwidth and signing, not a source of truth. Two kinds of merge exist, and the join is the law for one of them. Within a store, rows are folded by addition: the install's own sheet (\code{internal/pool/tally}) adds each of its rows once, as it is judged, and the reference relay adds each row it receives into the sender's triple for that cell and day. Addition is not idempotent, so a batch retried after a lost acknowledgement can be counted twice there; the relay bounds that by its daily row quota per install, and a relay that wants exactness keeps the batch's nonce. Between stores, when a relay, a mirror or an install holds a copy of another's sheet, the join~\eqref{eq:join} is the merge, and Proposition~\ref{prop:crdt} is what makes copies converge whatever the order. + +\subsection{Judges are not the same ruler} + +Let $y$ be a score of role $r$, model $m$, by judge $j$. We posit the two-way additive model +\begin{equation} +y=\mu_{rm}+\beta_j+\varepsilon,\qquad \E\varepsilon=0,\ \operatorname{Var}\varepsilon=\sigma^2, +\label{eq:twoway} +\end{equation} +with $\mu_{rm}$ the quantity of interest and $\beta_j$ the judge's severity, constrained by $\sum_j n_{\cdot\cdot j}\beta_j=0$ (the judges' weighted mean is the ruler). Weighted least squares needs only the cell means $\bar y_{rmj}=S_{rmj}/n_{rmj}$ and counts. The normal equations are solved by alternating +\begin{equation} +\beta_j\leftarrow\frac{\sum_{r,m} n_{rmj}(\bar y_{rmj}-\mu_{rm})}{\sum_{r,m} n_{rmj}},\qquad +\mu_{rm}\leftarrow\frac{\sum_{j} n_{rmj}(\bar y_{rmj}-\beta_j)}{\sum_{j} n_{rmj}}, +\label{eq:als} +\end{equation} +re-centring $\beta$ after each sweep. Each half-step minimises a convex quadratic in its block, so the residual is non-increasing and the iteration converges to the least-squares solution; five sweeps suffice in practice. + +\begin{proposition}[Identifiability] +\label{prop:ident} +Let $G$ be the bipartite graph with the judges on one side, the pairs $(r,m)$ on the other, and an edge where $n_{rmj}>0$. The parameters of~\eqref{eq:twoway} are identified up to one additive constant per connected component of $G$: within a component every contrast is identified, and one centring constraint per component fixes its levels. When $G$ is connected the single constraint identifies everything. +\end{proposition} +\begin{proof} +The design matrix of a two-way additive model has rank equal to (number of vertices of $G$) minus (number of connected components), and its kernel is spanned by the vectors that shift $\mu$ up and $\beta$ down by one constant on one component; one linear constraint per component removes them. With the single centring constraint, exactly one component is fully identified. +\end{proof} + +The reference relay centres $\beta$ by one weighted mean, which fixes the levels only when $G$ is connected; on a disconnected graph the between-component offsets are a convention of the fit, and the index says which judges each cell was scored by so a reader can tell. The judge rule of Section~5.1 makes $G$ connected quickly in practice: it sends most installs to the same few cheap judges. A model scored by a single judge so far has $\mu_{rm}$ equal to its own mean and the index says so. + +\subsection{Installs are not independent, and some are wrong} + +Rows from one install share a task mix, a codebase and a way of writing briefs. Under an intra-class correlation $\varrho$, $n_u$ rows from install $u$ carry the information of $n_u/(1+(n_u-1)\varrho)$ independent ones, which saturates at $1/\varrho$. The pool's cell estimate therefore treats installs, not rows, as the unit: it forms the install means $\bar y_{u\theta}$ and combines them with a location estimator that is robust to a minority of installs that are broken or adversarial. We use the Huber $M$-estimate with scale $s=1.4826\cdot\mathrm{MAD}$ and the conventional tuning constant $1.345$ (95\% efficiency at the normal), which has breakdown point $1/2$ in location, and fall back to the precision-weighted mean when a cell has fewer than five installs. A cell backed by fewer than three installs is published but not read. + +\subsection{Drift} + +Models change under the same name. Let $\bar y_d$ be a cell's day-$d$ mean over $n_d$ rows and model the true level as a random walk, +\begin{equation} +\mu_d=\mu_{d-1}+\eta_d,\quad \eta_d\sim\mathcal N(0,\omega^2);\qquad \bar y_d=\mu_d+\bar\varepsilon_d,\quad \bar\varepsilon_d\sim\mathcal N(0,\sigma^2/n_d). +\label{eq:localevel} +\end{equation} +The Kalman filter for this local-level model is, in steady state with constant $n_d=\bar n$, an exponentially weighted average whose forgetting factor $1-\alpha$ solves $\alpha^2/(1-\alpha)=\bar n\omega^2/\sigma^2$; the effective memory is about $1/\alpha$ days. The drift variance is estimated by moments from the day-to-day differences, +\begin{equation} +\hat\omega^2=\Big[\widehat{\operatorname{Var}}(\bar y_d-\bar y_{d-1})-\sigma^2\big(\tfrac1{n_d}+\tfrac1{n_{d-1}}\big)\Big]_+, +\label{eq:drift} +\end{equation} +pooled across cells. A cell with no evidence of drift ($\hat\omega^2=0$) keeps everything; a cell that drifts forgets at the rate the data show. The day is a cell dimension for exactly this reason. + +\begin{figure}[t] +\centering +\begin{tikzpicture}[x=1cm,y=1cm, + box/.style={draw,rounded corners=2pt,inner sep=4pt,font=\small,align=center,minimum height=8mm}, + store/.style={box,fill=black!6}, + net/.style={box,fill=blue!6}, + arr/.style={-{Latex[length=2mm]},thick}, + lbl/.style={font=\scriptsize,fill=white,inner sep=1.5pt} +] +\node[box] (cat) at (1.4,0) {catalog\\prices, indexes}; +\node[store] (post) at (4.7,0) {posterior per\\(role, model)}; +\node[box] (ts) at (8.1,0) {Thompson draw\\$\argmin_\kappa \tilde E(\kappa)$}; +\node[box] (run) at (10.7,0) {run}; +\node[box] (judge) at (13.3,0) {judge\\outside the crew}; +\node[store] (outbox) at (10.7,-1.6) {outbox}; +\node[store] (own) at (13.3,-1.6) {own sheet}; +\node[store] (index) at (4.7,-4.1) {signed index\\(mirrored)}; +\node[net] (relay) at (9.4,-4.1) {relay: fold, judge effects \eqref{eq:als},\\Huber over installs, sign}; +\node[box,dashed] (others) at (13.3,-4.1) {other installs}; +\draw[arr] (cat) -- (post); +\draw[arr] (post) -- (ts); +\draw[arr] (ts) -- (run); +\draw[arr] (run) -- (judge); +\draw[arr] (judge) -- (own); +\draw[arr] ([xshift=-7mm]judge.south) |- (outbox.east); +\draw[arr] (own.south) -- ++(0,-0.45) -| node[lbl,pos=0.12,below]{at once} (post.south); +\draw[arr] (outbox.south) -- node[lbl,right]{rows, no text} (outbox.south |- relay.north); +\draw[arr] (relay) -- (index); +\draw[arr] (index.north) -- node[lbl,pos=0.35,right]{read at start} (post.south); +\draw[arr] (others.west) -- node[lbl,above]{rows} (relay.east); +\draw[arr] (index.south) -- ++(0,-0.5) -| node[lbl,pos=0.25,below]{the same index} (others.south); +\draw[dotted,thick] (-0.1,0.75) rectangle (14.9,-2.55); +\node[font=\scriptsize] at (0.9,0.95) {one install}; +\end{tikzpicture} +\caption{Where each step runs. Inside the dotted box is one install; the only thing that crosses it outward is a row without text. The relay merges publications with the join of Proposition~\ref{prop:crdt} and publishes a signed index; any install can host a relay, and any node that kept the publications can rebuild one.} +\label{fig:flow} +\end{figure} + +\section{Estimation with no tuned constant} +\label{sec:eb} + +We now combine the three sources for one install $u$. Write $\hat q_r(m)$ for the catalog quality~\eqref{eq:catq}, $\bar y^{\mathrm{pool}}_{rm}$ and $n^{\mathrm{pool}}_{rm}$ for the pool's cell after Sections~5.3--5.5, and $\bar y^{u}_{rm}$, $n^{u}_{rm}$ for the own sheet. + +\begin{assumption}[Hierarchy] +\begin{equation} +\mu_{rm}\sim\mathcal N\big(\hat q_r(m)+b,\,\tau_0^2\big),\qquad +\mu^{u}_{rm}\mid\mu_{rm}\sim\mathcal N\big(\mu_{rm},\,\tau_1^2\big),\qquad +y\mid\mu^{u}_{rm}\sim\mathcal N\big(\mu^{u}_{rm},\,\sigma^2\big). +\label{eq:hier} +\end{equation} +\end{assumption} +The first line says the catalog predicts the population quality up to one offset $b$ shared by every model (the catalog's scale is not the judge's) and a noise $\tau_0$; the second that install $u$'s own quality for the model scatters around the population's; the third is judge noise after Section~5.3. The install cares about $\mu^u_{rm}$. + +\begin{proposition}[Posterior for the install] +\label{prop:post} +Under~\eqref{eq:hier}, with $k=\sigma^2/\tau_0^2$, $\ell=\sigma^2/\tau_1^2$ and $\tilde n=n^{\mathrm{pool}}\,\ell/(\ell+1)$ (pool rows are other installs' rows, so each carries variance $\sigma^2+\tau_1^2$ about $\mu_{rm}$), the posterior of $\mu^u_{rm}$ is normal with +\begin{equation} +\E[\mu^u_{rm}\mid\cdot]=\frac{k^{\ast}\,(\hat q_r(m)+b) + n^{\ast}\,\bar y^{\mathrm{pool}}_{rm} + n^u\,\bar y^u_{rm}}{k^{\ast}+n^{\ast}+n^u},\qquad +\operatorname{Var}[\mu^u_{rm}\mid\cdot]=\frac{\sigma^2}{k^{\ast}+n^{\ast}+n^u}, +\label{eq:three} +\end{equation} +where the catalog and the pool share one \emph{capped} external count $W$ in proportion to what each contributed, +\begin{equation} +W=\frac{K\,\ell}{K+\ell}\ \le\ \ell,\qquad K=k+\tilde n,\qquad +k^{\ast}=\frac{k}{K}\,W,\qquad n^{\ast}=\frac{\tilde n}{K}\,W . +\label{eq:cap} +\end{equation} +\end{proposition} +\begin{proof} +Integrate out $\mu_{rm}$. The catalog and the pool inform $\mu^u_{rm}$ only through $\mu_{rm}$, whose posterior given them is normal with precision $K/\sigma^2$, $K=k+\tilde n$, and mean the $k:\tilde n$ precision-weighted average of $\hat q_r(m)+b$ and $\bar y^{\mathrm{pool}}_{rm}$; adding the between-install variance $\tau_1^2$ gives, for $\mu^u_{rm}$, a precision $1/(\sigma^2/K+\tau_1^2)=W/\sigma^2$ with the same mean. The own rows add precision $n^u/\sigma^2$ directly. Precision-weighting the two gives~\eqref{eq:three}, with the external count $W$ split as $k^{\ast}+n^{\ast}=W$ in the ratio $k:\tilde n$. +\end{proof} + +\begin{corollary} +With $\tau_1=0$ ($\ell=\infty$) the posterior mean is $(1-w)\hat q_r(m)+w\bar y$ with $w=n/(n+k)$, $n=n^{\mathrm{pool}}+n^u$: the plain shrinkage blend, which is what the picker applies today with $k=30$. +\end{corollary} + +The cap~\eqref{eq:cap} is the point of the hierarchy. However many rows the pool holds, everything external to this install, catalog and pool together, is worth at most $\ell=\sigma^2/\tau_1^2$ own rows, because other installs' tasks are not this install's tasks; and the own rows are never capped. This replaces an ad hoc "own rows count triple" by a quantity with a meaning and an estimator. The pick shipped today is the Corollary's form: the pool's and the install's cells are folded together by observation count against the catalog at $k=30$, which is~\eqref{eq:three} at $\ell=\infty$ and $b=0$. + +\paragraph{Estimating the constants.} +Every constant in~\eqref{eq:three} is estimated from the sheet by moments; none is set by hand. +\begin{align} +\hat\sigma^2 &= \frac{\sum_{u,\theta}\big(S^2_{u\theta}-S_{u\theta}^2/n_{u\theta}\big)}{\sum_{u,\theta}(n_{u\theta}-1)} &&\text{pooled within-cell variance,}\label{eq:sigma}\\ +\hat\tau_1^2 &= \Big[\,\widehat{\operatorname{Var}}_u\big(\bar y_{u\theta}\big)-\hat\sigma^2\,\overline{(1/n_{u\theta})}\,\Big]_+ &&\text{between-install variance, pooled over cells,}\label{eq:tau1}\\ +\hat b &= \overline{\bar y^{\mathrm{pool}}_{rm}-\hat q_r(m)} &&\text{the catalog's offset,}\label{eq:bias}\\ +\hat\tau_0^2 &= \Big[\,\widehat{\operatorname{Var}}_{(r,m)}\big(\bar y^{\mathrm{pool}}_{rm}-\hat q_r(m)\big)-\overline{(\hat\sigma^2+\hat\tau_1^2)/n^{\mathrm{pool}}_{rm}}\,\Big]_+ &&\text{how far the catalog misses,}\label{eq:tau0} +\end{align} +each a within-minus-noise moment estimator of the kind used by Efron and Morris~\cite{efron1975}. Then $k=\hat\sigma^2/\hat\tau_0^2$ and $\ell=\hat\sigma^2/\hat\tau_1^2$, and $\hat\omega^2$ of~\eqref{eq:drift} sets the forgetting. On the seed index alone the estimator already says something: its thirteen cells with a published intelligence index give $\hat b=-13.7$ (the judged scale sits 14 points below the catalog's) and a variance of the residuals, 109, below the noise they carry, 167, so $\hat\tau_0^2$ clips to zero: thirteen cells of bit-valued scores ($\hat\sigma\approx43$) cannot resolve a catalog error under about 13 points, and the day-one $k=30$ stands until judged scores, with their smaller $\sigma$, replace the bits. The design publishes them in the index beside the cells so that every install blends with the same constants, re-estimated at every relay cycle and withheld until each denominator has at least ten degrees of freedom, with the day-one values of Table~\ref{tab:const} until then; the index on this head carries the cells and the day-one constants, and the estimators are the next thing the relay's job runs. + +\subsection{Covariates, and why no row is ever wasted} +\label{sec:covariates} + +A row carries more than (role, model, score): a door, a size bucket, a judge, a day, an install, and any field added later, such as the harness version or a task family. Call the vector of such fields the row's \emph{profile} $c\in\mathcal C=\prod_f L_f$, a product of factor levels. The hierarchy~\eqref{eq:hier} extends factor by factor: +\begin{equation} +\mu_{rm,c}=\mu_{rm}+\sum_{f}\delta^{f}_{rm,c_f},\qquad \delta^{f}_{rm,\cdot}\sim\mathcal N(0,\tau_f^2)\ \text{i.i.d.\ over levels}, +\label{eq:factors} +\end{equation} +so that a (role, model) has one population quality and one partially pooled offset per level of each factor. The install factor is the case $f=\text{install}$ of~\eqref{eq:factors}, and Proposition~\ref{prop:post} is its two-level instance; the same derivation gives, for a query profile $c$, a posterior mean that is a precision-weighted combination of the cell means at every level that shares a coordinate with $c$, each capped by its own $\ell_f=\sigma^2/\tau_f^2$, with $\hat\tau_f^2$ estimated by the between-level moment~\eqref{eq:tau1} applied to factor $f$. Two properties follow. + +\emph{Old rows keep their value when the harness changes.} If the door algorithm is rewritten, rows from the old door are a level of the door factor; they still inform $\mu_{rm}$ with the full weight the estimated $\tau_{\text{door}}^2$ allows, and the new door starts at $\mu_{rm}$ plus nothing and specialises as its own rows arrive. If the two doors turn out to differ little, $\hat\tau_{\text{door}}^2\approx 0$, $\ell_{\text{door}}\to\infty$, and the factor pools completely; if they differ a lot, the cap keeps the old door from speaking for the new one. The data decide, per factor, how much they transfer. + +\emph{The schema is open.} A row with a field absent (older rows, before the field existed) is at a level of its own, \emph{unknown}, which the estimator treats like any other level. Adding a field therefore invalidates nothing already collected, and the sheet's join in Proposition~\ref{prop:crdt} is unaffected because a cell key is only a tuple. What can be extracted from the rows is the set of contrasts the factors expose; a factor that is never recorded cannot be recovered later, which is why the row carries every cheap field now and the text of none. + +Regressors that are not factors, a task's token count for instance, enter the same way as a linear term with a partially pooled slope; the moment estimators are the same with a design column in place of an indicator. + +\paragraph{Interactions: the crew as a group.} +Assumption~\ref{ass:linear} scores roles one at a time. Whether a particular worker and a particular checker do better \emph{together} than their role qualities predict is a question about the pair, and the same hierarchy answers it: a pair $(m_r,m_{r'})$ is a level of a factor over $\mods\times\mods$, with its own partially pooled offset $\delta^{rr'}_{(m_r,m_{r'})}\sim\mathcal N(0,\tau_{rr'}^2)$. A pair that has never been run sits at zero offset and the additive prediction stands; a pair with rows moves by exactly the amount its rows justify against $\hat\tau_{rr'}^2$; if pairs in general add nothing, $\hat\tau_{rr'}^2\to0$ and the term disappears. This is the whole difference between learning crews as groups and learning roles: the group term is there, but it is charged for. A crew-level learner with no structure has to visit each of $|\crews|$ crews to say anything about it; the additive model needs $\sum_r|S_r|$ observations to say something about every crew, and the pair term then spends further observations only where a pair is actually run. Some group effects are constraints rather than parameters: the vendor rule $D$ is one, and it is applied to $\crews$ before any learning happens. + +\begin{table}[h] +\centering\small +\begin{tabular}{p{2.0cm}p{5.2cm}p{4.0cm}p{2.6cm}}\toprule +constant & meaning & estimator & day-one value\\\midrule +$\sigma$ & judge noise & \eqref{eq:sigma} & 15\\ +$k=\sigma^2/\tau_0^2$ & weight of the catalog & \eqref{eq:tau0} & 30\\ +$\ell_f=\sigma^2/\tau_f^2$ & cap on evidence transferred across a factor (install, door, size, \dots) & \eqref{eq:tau1} per factor & 30\\ +$\omega^2$ & drift per day & \eqref{eq:drift} & 0 (no forgetting)\\ +$\beta_j$ & judge severity & \eqref{eq:als} & 0\\ +$v_r$ & role token shares & measured from run token accounting & $0.90/0.08/0.02$\\ +\midrule +$H$ & price of a shipped defect & the user's, by budget word & \$1 / \$10 / \$50\\ +$\lambda$ & price of a minute & the user's & \$0.01\\ +$\phi$, judge floor & structural floors & fixed by design & $0.8$, $60$\\ +\bottomrule\end{tabular} +\caption{Every constant with its estimator. Two are the user's by design; two are structural floors. The rest are empirical-Bayes estimates that the relay publishes with the index.} +\label{tab:const} +\end{table} + +\begin{figure}[t] +\centering\includegraphics{fig-shrink.pdf} +\caption{Shrinkage of one role quality from the catalog prior (80) toward the measured mean (65), $k=30$, $\sigma=15$. The band is what the Thompson draw of Section~\ref{sec:decide} samples from.} +\label{fig:shrink} +\end{figure} + +\section{Deciding: Thompson sampling with semi-bandit feedback} +\label{sec:decide} + +Given a posterior $\mathcal N(\mu'_{rm},\sigma'^2_{rm})$ per role and model from~\eqref{eq:three}, the pick could minimise~\eqref{eq:E} at the posterior means. That never tries a crew the prior underrates, and the catalog prior is wrong by a lot for some models (Figure~\ref{fig:cells}). Thompson sampling~\cite{thompson1933,russo2018} draws one plausible world from the posterior and acts as if it were true: a crew is chosen with the probability that it is the best, exploration is proportional to what is unknown, and it stops on its own as the posteriors tighten. No exploration rate exists to be set. + +\begin{algorithm}[h] +\caption{One pick under \emph{learn}} +\begin{algorithmic}[1] +\Require shortlists $S_r$ on the posterior means, budget word $\Rightarrow H$, task size bucket $\Rightarrow$ expected tokens $T$ and minutes $t(\kappa)$ +\For{each role $r$ and $m\in S_r$} + \State $\tilde q_{rm}\sim\mathcal N(\mu'_{rm},\sigma'^2_{rm})$ \Comment{one draw per run; never redrawn during the run} +\EndFor +\For{each $\kappa\in\crews$} + \State $\tilde E(\kappa)\gets T\,B(\kappa)+\lambda\,t(\kappa)+H\,\big(1-\sum_r\nu_r\tilde q_{r\kappa(r)}/100\big)$ +\EndFor +\State \Return $\argmin_\kappa\tilde E(\kappa)$ \Comment{ties: fewer imputed indexes, then id order} +\end{algorithmic} +\end{algorithm} + +\begin{proposition}[Role draws are crew draws] +Under Assumption~\ref{ass:linear}, drawing $\tilde q_{rm}$ independently per (role, model) and minimising $\tilde E$ over $\crews$ is Thompson sampling over the crews' posterior costs under the product posterior on role qualities. +\end{proposition} +\begin{proof} +$\tilde E(\kappa)$ is an affine function of the vector $(\tilde q_{rm})$, so its joint law over $\crews$ is the push-forward of the posterior on role qualities; a sample of the latter is a sample of the former. The judge-effect fit of Section~5.3 and the drift model of Section~5.5 induce dependence between cells that the independent draw ignores; the draw is exact for the product approximation the index publishes. +\end{proof} + +The structure that matters is the feedback. A run of crew $\kappa$ is judged role by role, so it yields one observation for each of its $n$ (role, model) pairs, and the posterior of every crew that shares a pair with $\kappa$ moves. This is a combinatorial bandit with semi-bandit feedback~\cite{chen2013,wen2015}: the unknowns are the $\sum_r|S_r|$ role qualities, not the $|\crews|$ crew values, and the regret of Thompson sampling scales with the former~\cite{wen2015}. Section~\ref{sec:exp} measures the difference on our runs. For Gaussian posteriors the Bayesian regret of Thompson sampling is $O(\sqrt{dT\log T})$ in the dimension $d$ of the unknowns over $T$ runs~\cite{russo2014}. + +\paragraph{The learned front.} +Under \emph{learn} the front of Section~\ref{sec:catalog} is recomputed on the posterior: the bill $B(\kappa)$ is read at the run's expected tokens, and the quality is the posterior mean of $\pclean(\kappa)$, so the front is the pareto set in (expected dollars per task, expected probability of a clean delivery) with an interval on each point from~\eqref{eq:three}. The budget word's $H$ is a line of slope $-1/H$ in that plane and the pick is where the line first touches the front; Algorithm~1 is the same pick on one draw of the front rather than on its mean, which is what makes it explore. The pool command prints the mean front with its intervals, and the catalog front beside it, so that a user can see where measurement moved a pick and by how much. + +\paragraph{When is it settled.} +The same posterior answers whether learning is done: the probability $\pi^\ast$ that the posterior-mean crew is the best, estimated by repeating the draw of Algorithm~1 a few hundred times. The pool command reports $\pi^\ast$; a value near one means further exploration is nearly free, a value near $1/|\front|$ means the ranking is still open. Costs enter as observed dollars per run and are not sampled; their noise is small next to the quality's. + +\begin{remark}[A new model] +A model with no rows has $n=0$ and sits at its catalog prior with the prior variance $\sigma^2/k^{\ast}$. If that puts it near the front it is drawn into crews soon, judged, and confirmed or pushed down by its first rows. Nothing is done by hand when the catalog changes. +\end{remark} + +\section{Experiments on our measured runs} +\label{sec:exp} + +Ground truth is the table of Figure~\ref{fig:front}: 56 crews with $\pclean$, dollars and minutes fitted by a ridge model on 72 scored runs of our own over nine tasks. Each replicate draws a catalog prior per arm as the truth plus $\mathcal N(0,0.15^2)$ noise, roughly the disagreement between indexes and measurements in Figure~\ref{fig:cells}, holds it with $k=30$, and runs $N$ tasks; a strategy picks, observes the outcome and pays~\eqref{eq:E} with $H=\$10$. Regret is the excess over the oracle crew, averaged over 200 replicates. + +\paragraph{Crew-level learners.} +Table~\ref{tab:regret} and Figure~\ref{fig:regret} compare, on all 23 chat-door crews of the table, a fixed table row (the shipped balanced crew), the catalog prior alone, $\varepsilon$-greedy and Thompson, all with one arm per crew. After 400 runs Thompson is \$0.33 per task under the fixed table, a third of the table's regret, and still falling; $\varepsilon$-greedy is \$0.12 behind it. Thompson pays an exploration tax in its first forty runs, when it is no better than the table; $k=30$ is what bounds it. $\varepsilon$-greedy pays more, because a tenth of its runs go to a random crew among 23, and its rate neither grows when a new model appears nor shrinks when the ranking is settled. + +\begin{table}[h] +\centering\small +\input{tab-regret.tex} +\caption{Regret against the oracle, dollars per task after $N$ runs, one arm per crew.} +\label{tab:regret} +\end{table} + +\begin{figure}[h] +\centering\includegraphics{fig-regret.pdf} +\caption{Left: regret per task. Right: share of runs on a crew other than the current posterior-mean best. Thompson explores heavily while many crews sit within cents of each other and less as the posteriors separate; $\varepsilon$-greedy explores at its fixed rate.} +\label{fig:regret} +\end{figure} + +\paragraph{Role-level feedback.} +Figure~\ref{fig:semibandit} and Table~\ref{tab:semibandit} isolate the structure of Section~\ref{sec:decide}. The ground truth is three-role: a role quality per (role, model) read off the measured rows (a worker's delivered rate, a checker's and a planner's clean rate), combined by the linear rule of Assumption~\ref{ass:linear} over every crew the vendor rule admits, with dollars additive over roles. Three learners run Thompson sampling with the same prior noise and the same $H$ and differ only in what they see after a run: one arm per crew fed the run's clean-or-defect bit; one arm per crew fed the mean of the judge's role scores; one posterior per (role, model) fed each role's own score, which is what the judge of Section~5.1 provides. The last is the semi-bandit. Richer feedback per run helps: the crew-level learner fed the mean score sits at \$0.31 after four hundred runs against \$0.54 for the one fed only the clean bit. The structure helps again as much: the role-level learner, fed the same scores as the second, sits at \$0.15, half of it and a third of the first, because every run updates the crews that share a model with the one that ran, and there are sixteen unknowns behind 138 crews. + +\begin{table}[h] +\centering\small +\input{tab-semibandit.tex} +\caption{Regret against the oracle, dollars per task after $N$ runs, on the three-role truth; same prior noise and $H$ for all three learners.} +\label{tab:semibandit} +\end{table} + +\begin{figure}[h] +\centering\includegraphics{fig-semibandit.pdf} +\caption{Thompson sampling under three kinds of feedback on the same three-role truth. Dotted and dashed: one arm per crew, fed the clean bit and the mean role score respectively. Solid: one posterior per (role, model) fed the role scores, the combinatorial semi-bandit of Section~\ref{sec:decide}.} +\label{fig:semibandit} +\end{figure} + +\begin{figure}[t] +\centering\includegraphics{fig-cells.pdf} +\caption{The seed index shipped in the binary: 15 cells from 194 scored runs, \code{role\_quality} $=100\cdot P(\text{no major defect})$ under a Beta(1,1) posterior. The spread between models a catalog rates alike is why the pool exists.} +\label{fig:cells} +\end{figure} + +The truth in these experiments is our own fitted table from one team's tasks, so the size of the gain is the shape of the effect and not a promise for every install; the regret scale is set by how close the crews sit, which the front of Figure~\ref{fig:front} shows and no bound here uses. The script \code{figures.py} beside this document reproduces every figure and table from the two CSV files it reads and the constants written at its head. + +\section{System architecture} +\label{sec:arch} + +The statistics of Sections~\ref{sec:pool}--\ref{sec:decide} fix what the system computes; this section fixes where the bytes live, so that the privacy claims can be checked against the deployment rather than taken on trust. Figure~\ref{fig:arch} shows the four components. The design rule is that the relay holds nothing worth stealing and nothing that cannot be rebuilt. + +\paragraph{The binary.} +Everything that touches a task runs inside the install: the catalog read, the posterior, the draw, the run, the judge call, the own sheet and the outbox. The judge is an ordinary model call through the install's own provider credentials; the record it reads never leaves the machines the install already trusts with its work. The outbox is a newline-delimited file under the profile directory; \code{codeaf pool status} prints its pending rows, and a user who wants to see exactly what would be sent can read the file. The loader reads the fresher of the cached index and the embedded seed index, by generated day, once at start-up; a refresh runs in the background and takes effect at the next start, so a running task's picks never move under it. + +\paragraph{The relay.} +The reference relay is a single edge function with an object store behind it, deployable in minutes on a commodity edge platform (\code{wrangler deploy} from \code{relay/} in the repository; the runbook beside this document is ten steps). It exposes two routes. \code{POST /v1/rows} accepts a batch of rows, validates the schema, folds each row into the sender's triple for its cell and day, and discards the row: the store holds the sheet of Definition~2 and nothing else, one triple per (install, cell, day), so its size is bounded by installs times cells times days and not by traffic, and a row that has been folded cannot be recovered from it. \code{GET /index.json} serves the last published index and its detached signature. An hourly scheduled job reads the sheet, runs the judge-effect fit and the robust aggregation of Section~5 (the drift estimate of Section~5.5 is specified and next to land), writes the index, signs it with an Ed25519 key held as a platform secret, and mirrors the pair of files to a public git branch so that they are browsable, diffable and forkable. The function keeps no request logs and stores no network address; the install nonce is the only identity in the store, and rate limiting is by nonce (a cap on rows per day) rather than by address. + +\paragraph{Priming, and one document at a time.} +The pool does not start empty. The same measured runs the seed of Figure~\ref{fig:cells} encodes were loaded once as ordinary rows, by a committed script in \code{relay/tools}, under one install nonce per originating session and one judge id for the reviewer that scored them; so the first published index carries the cells the binary already knows, and an install that reaches the relay reads numbers on day one rather than an empty document. Nothing is counted twice, because an install holds exactly one index document at a time: the cached one when it parses and is not older, by its generated day, than the seed the binary carries, and the seed otherwise. The seed is the offline fallback, never a second source blended with the fetched one. That the priming rows were scored by review and later rows by a judge model is the judge-severity situation of Section~\ref{sec:pool} already: one more judge id, fitted like the others. + +\paragraph{Keys and verification.} +The signing key's public half is compiled into the binary as a list, so a key can be rotated by shipping a binary that carries both; the private half exists only in the platform secret store. Verification is client-side: the binary checks the signature before it reads a single cell, refuses a version lower than the one it holds (a rollback is caught by the version; a replayed old index is refused the same way), and on any failure keeps the cached copy beside an error. Because verification is local, the index can be served from anywhere: the mirror, a static file host, or a directory on disk, which is what the \code{file://} address is for. + +\paragraph{Trust and failure.} +What a compromised relay learns is the sheet: counts, sums and sums of squares of scores per (install nonce, role, model, judge, day, size, door). It learns no task, no code, no prompt, no user, and no address. What a compromised relay can do is publish a bad index, which the signature stops unless the key is also taken; a taken key is retired by shipping the next key, and the min-installs and robust-aggregation rules bound what a flood of fabricated nonces can move in the meantime. What an unreachable relay costs is nothing at all: the cache, then the seed, answer every read, and the outbox waits. Any install can run the same function against its own store and point its \code{CODEAF\_MODEL\_POOL\_URL} and \code{\_SUBMIT\_URL} at it; two relays that receive the same publications produce the same sheet (Proposition~\ref{prop:crdt}), so a private relay and the public one can be federated later by a one-way copy of publications with no reconciliation step. + +\begin{figure}[H] +\centering +\resizebox{\textwidth}{!}{% +\begin{tikzpicture}[x=1cm,y=1cm, + box/.style={draw,rounded corners=2pt,inner sep=4pt,font=\small,align=center,minimum height=8mm}, + store/.style={box,fill=black!6}, + net/.style={box,fill=blue!6}, + arr/.style={-{Latex[length=2mm]},thick}, + lbl/.style={font=\scriptsize,fill=white,inner sep=1.5pt} +] +\node[box,text width=3.2cm] (bin) at (2.0,0) {binary\\posterior, draw, run, judge}; +\node[store,text width=3.2cm] (outbox) at (2.0,-1.8) {outbox\\rows, local file}; +\node[store,text width=3.2cm] (cache) at (2.0,-3.6) {index cache\\and embedded seed}; +\node[net,text width=6.1cm] (fn) at (8.2,-1.8) {edge function\\\code{POST /v1/rows}: fold a row into its triple\\\code{GET /index.json}: serve the last index}; +\node[store,text width=6.1cm] (kv) at (8.2,-4.2) {object store\\one triple per (install, cell, day)}; +\node[net,text width=3.4cm] (cron) at (13.7,-4.2) {hourly job\\fold, \eqref{eq:als}, Huber, sign}; +\node[store,text width=3.4cm] (mirror) at (13.7,-1.8) {public mirror\\\code{index.json} + \code{.sig}}; +\node[box,dashed,text width=3.4cm] (secret) at (13.7,-6.0) {platform secret\\Ed25519 private key}; +\draw[arr] (bin) -- (outbox); +\draw[arr] (cache.west) -- ++(-0.4,0) |- node[lbl,pos=0.25,left]{read at start} (bin.west); +\draw[arr] (outbox.east) -- node[lbl,above]{https, batched} (fn.west |- outbox.east); +\draw[arr] (fn) -- node[lbl,right]{fold} (kv); +\draw[arr] (kv) -- node[lbl,above]{sheet} (cron); +\draw[arr] (cron) -- node[lbl,right]{signed index} (mirror); +\draw[arr] (secret) -- (cron); +\draw[arr] (mirror.west) -- node[lbl,below,pos=0.6]{pull, verify} (fn.east |- mirror.west); +\draw[arr] ([xshift=-2.2cm]fn.south) |- node[lbl,pos=0.75,above]{cached index} (cache.east); +\node[font=\scriptsize] at (8.0,0.2) {no request logs, no addresses stored, rows discarded after folding}; +\end{tikzpicture}} +\caption{Deployment. The relay is one edge function, one object store holding only the sheet, and one scheduled signing job; the install verifies every index locally and never blocks on the network. Any of the three network components can be replaced by a static file, a directory, or a self-hosted copy.} +\label{fig:arch} +\end{figure} + +\section{What the user controls} + +\begin{center}\small +\begin{tabular}{p{2.1cm}p{4.3cm}p{8.0cm}}\toprule +setting & values & meaning\\\midrule +\code{crew} & frugal $\mid$ balanced $\mid$ max $\mid$ custom & how much to spend: $H$ and the knob $k$\\ +\code{picked from} & table $\mid$ catalog $\mid$ learn & the rows we measured; recomputed from today's catalog; catalog plus pool plus own sheet\\ +\code{model\_pool} & on $\mid$ read $\mid$ off & read the index and send rows (default); read only; neither\\ +\code{explore} & on $\mid$ off & Thompson draw (default) or the posterior mean; \emph{learn} only; specified, not on this head\\ +\bottomrule\end{tabular} +\end{center} + +With the pool set to \emph{read}, the own sheet still accrues and enters~\eqref{eq:three}; with it \emph{off}, only the catalog prior remains and the pick is the catalog's. The index is a JSON document signed with an Ed25519 key whose public half is in the binary; \code{codeaf pool verify} checks it, \code{pool show} prints its summary and each address with its source, \code{pool status} counts the rows waiting in the outbox and says whether the relay answered. The binary carries a seed index of our own scored runs (Figure~\ref{fig:cells}) so that \emph{learn} has numbers on day one. The relay's code is in this repository; its addresses are settings, so an install can read from and send to a relay of its own, or to none. + +\section{Status} + +On the head this document ships with: the catalog crew of Section~\ref{sec:catalog}; the shrinkage blend of the Corollary with $k=30$; the \code{picked from} row and its ladder; the index document, its reader and signed pull with the built-in key; the outbox, the install nonce and the push; the judge, its deterministic choice, and the hook that calls it when a chat task lands; the own sheet, whose cells enter the prior beside the index's by observation count; the settings; the seed index; the relay of Section~\ref{sec:arch} with its store keyed by install and day, the join, the judge-effect fit and the Huber aggregation; the mirror workflow. Specified here and next to land: the drift estimate~\eqref{eq:drift}; the hierarchical cap~\eqref{eq:cap} with the moment estimators~\eqref{eq:sigma}--\eqref{eq:tau0} in place of the count fold; the Thompson draw with role-level updates, the \code{explore} setting and the settled probability $\pi^\ast$; the pair term of Section~6.1; the headless doors' hook. The referee's report on an earlier draft, and what changed in answer to it, are beside this file. + +\begin{thebibliography}{9}\small +\bibitem{thompson1933} W.~R. Thompson. On the likelihood that one unknown probability exceeds another in view of the evidence of two samples. \emph{Biometrika} 25:285--294, 1933. +\bibitem{russo2018} D.~Russo, B.~Van Roy, A.~Kazerouni, I.~Osband, Z.~Wen. A tutorial on Thompson sampling. \emph{Foundations and Trends in Machine Learning} 11(1):1--96, 2018. +\bibitem{russo2014} D.~Russo, B.~Van Roy. Learning to optimize via posterior sampling. \emph{Mathematics of Operations Research} 39(4):1221--1243, 2014. +\bibitem{chen2013} W.~Chen, Y.~Wang, Y.~Yuan. Combinatorial multi-armed bandit: general framework and applications. \emph{ICML}, 2013. +\bibitem{wen2015} Z.~Wen, B.~Kveton, A.~Ashkan. Efficient learning in large-scale combinatorial semi-bandits. \emph{ICML}, 2015. +\bibitem{efron1975} B.~Efron, C.~Morris. Data analysis using Stein's estimator and its generalizations. \emph{Journal of the American Statistical Association} 70(350):311--319, 1975. +\bibitem{shapiro2011} M.~Shapiro, N.~Pregui\c{c}a, C.~Baquero, M.~Zawirski. Conflict-free replicated data types. \emph{SSS}, 2011. +\bibitem{huber1964} P.~J. Huber. Robust estimation of a location parameter. \emph{Annals of Mathematical Statistics} 35(1):73--101, 1964. +\bibitem{harvey1989} A.~C. Harvey. \emph{Forecasting, Structural Time Series Models and the Kalman Filter}. Cambridge University Press, 1989. +\end{thebibliography} + +\end{document} diff --git a/docs/design/model-pool/tab-regret.tex b/docs/design/model-pool/tab-regret.tex new file mode 100644 index 0000000000..463ceb71f2 --- /dev/null +++ b/docs/design/model-pool/tab-regret.tex @@ -0,0 +1,8 @@ +% generated by figures.py from data/measured-crews.csv; do not edit +\begin{tabular}{lrrrrrrrrr}\toprule +$N$ & 10 & 20 & 40 & 70 & 100 & 150 & 200 & 300 & 400\\\midrule +table & 1.01 & 0.95 & 0.95 & 0.93 & 0.95 & 0.93 & 0.94 & 0.96 & 0.97\\ +catalog & 0.99 & 0.94 & 0.94 & 0.91 & 0.92 & 0.88 & 0.88 & 0.90 & 0.91\\ +greedy & 1.20 & 1.10 & 1.01 & 0.99 & 0.94 & 0.90 & 0.86 & 0.80 & 0.76\\ +thompson & 0.97 & 0.97 & 0.93 & 0.87 & 0.88 & 0.84 & 0.79 & 0.71 & 0.64\\ +\bottomrule\end{tabular} diff --git a/docs/design/model-pool/tab-semibandit.tex b/docs/design/model-pool/tab-semibandit.tex new file mode 100644 index 0000000000..5a7ac2a294 --- /dev/null +++ b/docs/design/model-pool/tab-semibandit.tex @@ -0,0 +1,7 @@ +% generated by figures.py from data/measured-crews.csv; do not edit +\begin{tabular}{lrrrrrrrrr}\toprule +$N$ & 10 & 20 & 40 & 70 & 100 & 150 & 200 & 300 & 400\\\midrule +crew, clean bit & 0.47 & 0.50 & 0.50 & 0.51 & 0.53 & 0.54 & 0.55 & 0.55 & 0.54\\ +crew, mean score & 0.29 & 0.43 & 0.39 & 0.38 & 0.36 & 0.35 & 0.34 & 0.34 & 0.31\\ +role, semi-bandit & 0.36 & 0.35 & 0.31 & 0.27 & 0.26 & 0.22 & 0.20 & 0.17 & 0.15\\ +\bottomrule\end{tabular} diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 198c4f8ac6..063a366b58 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -107,22 +107,29 @@ type Model struct { // tops the webapps board and sits mid-table on 3d has a real strength an // average would report as mediocrity. Zero means nobody published one. ArenaElo float64 `json:"arena_elo,omitempty"` - // IntelligenceIndex is the one published score in the catalog, carried - // verbatim and never computed here. + // The three Artificial Analysis scores the catalog keeps, carried verbatim + // and never computed here. // // OpenRouter's rows may carry a `benchmarks` block, and inside it an // `artificial_analysis` object with `intelligence_index`, `coding_index` // and `agentic_index` — Artificial Analysis's numbers, republished. 155 of // 528 rows had one on 2026-08-11. Zero means NOBODY published a score, and - // never a model that scored zero: a surface showing this must render the + // never a model that scored zero: a surface showing these must render the // zero as absence the way it renders an absent price. // - // Only the intelligence index is kept. The other two are the same source - // saying the same thing at a different angle, and a catalog row is not the - // place to hold a benchmark suite. - IntelligenceIndex float64 `json:"intelligence_index,omitempty"` - InputModalities []string `json:"input_modalities,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` + // All three are kept because they are read seat by seat: the agentic and + // coding indexes describe a long tool loop, the intelligence index a single + // reasoning call, and the seat asking decides which one it needs. + IntelligenceIndex float64 `json:"intelligence_index,omitempty"` + CodingIndex float64 `json:"coding_index,omitempty"` + AgenticIndex float64 `json:"agentic_index,omitempty"` + // OpenWeights says the row's weights are published — OpenRouter's + // `hugging_face_id`, kept as the one-word answer to whether the weights are + // public. A row cached before this field existed reads false, which every + // reader must take as unknown rather than closed. + OpenWeights bool `json:"open_weights,omitempty"` + InputModalities []string `json:"input_modalities,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` // Parameters is which request fields the provider says this model accepts — // OpenRouter's `supported_parameters`, lowercased and deduped. // @@ -255,6 +262,11 @@ type Catalog struct { // exists. [Catalog.rows] blocks on the future; [Catalog.rowsNow] reads this // and takes "not yet" for an answer. warm atomic.Pointer[rows] + // warmed is closed the moment that publish happens, and is the door + // [Catalog.Warmed] waits at. Nil on a catalog that resolved eagerly — + // there was never a warm to wait for — and on a zero catalog, where nothing + // is in flight and nothing will land. + warmed chan struct{} // blocking counts the questions asked through [Catalog.rows] — the door that // can wait. See [Catalog.BlockingReads]. blocking atomic.Int64 @@ -354,10 +366,14 @@ func loadOrFallback(ctx context.Context, options Options) (resolved *rows, err e // the surface is up, so the goroutine warms the value while the caller carries // on, and only a question that genuinely arrives first ever blocks. func LoadLazy(ctx context.Context, options Options) *Catalog { - resolved := &Catalog{} + resolved := &Catalog{warmed: make(chan struct{})} resolve := sync.OnceValue(func() *rows { loaded, _ := loadOrFallback(ctx, options) resolved.warm.Store(loaded) + // The wait door ([Catalog.Warmed]) reads the close, not the value, and + // the two land together so a caller that arrived between them would + // see the rows and still wait. + close(resolved.warmed) return loaded }) resolved.resolve = resolve @@ -471,6 +487,41 @@ func (c *Catalog) rowsNow() *rows { return c.warm.Load() } +// Warmed answers whether this catalog's rows have landed, waiting within the +// bound ctx carries for a lazily loaded one — from the disk cache or from the +// fetch, whichever wins — and answering false when the bound ends first. A +// catalog that resolved eagerly ([Load], [Refresh]) answers true at once, and +// a nil catalog answers false. +// +// It exists for a caller whose next step reads the rows through a seam that +// must not wait ([Catalog.ModelsNow], and the binaries that set config's +// AutoModels from it): a bounded wait turns "not yet" into "the rows" when the +// rows are a disk read away, without ever turning the caller into a fetch. The +// caller owns the bound; this only honors it. +func (c *Catalog) Warmed(ctx context.Context) bool { + if c == nil { + return false + } + if c.ready != nil { + return true + } + if c.warm.Load() != nil { + return true + } + if c.warmed == nil { + // Not a lazily loaded catalog: nothing is in flight and nothing will + // land, so a wait would only spend the caller's bound. What is in hand + // is the whole answer, and it is nothing. + return false + } + select { + case <-c.warmed: + return true + case <-ctx.Done(): + return false + } +} + // ModelsNow is the whole model list for a caller that MUST NOT WAIT, and nil // while a lazily loaded catalog is still warming. // @@ -1023,7 +1074,10 @@ func fetch(ctx context.Context, options Options) ([]Model, error) { CacheReadPrice: cacheRead, PriceUnknown: !promptOK || !completionOK, ArenaElo: arenaElo(item.Benchmarks), - IntelligenceIndex: intelligenceIndex(item.Benchmarks), + IntelligenceIndex: analysisScore(item.Benchmarks, "intelligence_index"), + CodingIndex: analysisScore(item.Benchmarks, "coding_index"), + AgenticIndex: analysisScore(item.Benchmarks, "agentic_index"), + OpenWeights: item.HuggingFaceID != "", InputModalities: cleanLowerList(item.Architecture.Input), OutputModalities: cleanLowerList(item.Architecture.Output), Parameters: cleanLowerList(item.SupportedParameters), @@ -1083,6 +1137,10 @@ type modelWire struct { SupportedEfforts []string `json:"supported_efforts"` DefaultEffort string `json:"default_effort"` } `json:"reasoning"` + // HuggingFaceID is the row's `hugging_face_id`, read for the one fact it + // carries — the weights behind the row are published — and not kept on the + // Model beside that fact. + HuggingFaceID string `json:"hugging_face_id"` // Benchmarks stays raw so its shape cannot break the row around it. It // carried an object beside a LIST on 2026-08-11 (`design_arena: []` next // to `artificial_analysis: {…}`), which is exactly the kind of thing that @@ -1090,24 +1148,28 @@ type modelWire struct { Benchmarks json.RawMessage `json:"benchmarks"` } -// intelligenceIndex digs the one published score out of a raw benchmarks -// block, and answers zero for every shape it does not recognize. Nothing here -// is allowed to fail loudly: a score is a nicety on a row, and a catalog that -// refused to load because a benchmark changed shape would have traded four -// hundred models for one number. -func intelligenceIndex(raw json.RawMessage) float64 { +// analysisScore digs one published Artificial Analysis score out of a raw +// benchmarks block by its field name, and answers zero for every shape it does +// not recognize. Nothing here is allowed to fail loudly: a score is a nicety on +// a row, and a catalog that refused to load because a benchmark changed shape +// would have traded four hundred models for one number. The block is decoded to +// raw messages and only the asked-for field parsed, so one drifted score costs +// itself and never its siblings. +func analysisScore(raw json.RawMessage, field string) float64 { if len(raw) == 0 { return 0 } var block struct { - ArtificialAnalysis struct { - IntelligenceIndex float64 `json:"intelligence_index"` - } `json:"artificial_analysis"` + ArtificialAnalysis map[string]json.RawMessage `json:"artificial_analysis"` } if json.Unmarshal(raw, &block) != nil { return 0 } - if score := block.ArtificialAnalysis.IntelligenceIndex; score > 0 { + var score float64 + if json.Unmarshal(block.ArtificialAnalysis[field], &score) != nil { + return 0 + } + if score > 0 { return score } return 0 diff --git a/internal/catalog/catalog_test.go b/internal/catalog/catalog_test.go index d980b28061..5121ec8376 100644 --- a/internal/catalog/catalog_test.go +++ b/internal/catalog/catalog_test.go @@ -2,9 +2,12 @@ package catalog import ( "context" + "encoding/json" "errors" "io" "net/http" + "os" + "path/filepath" "strings" "sync/atomic" "testing" @@ -198,6 +201,86 @@ const cachePricePayload = `{"data":[ "pricing":{"prompt":"-1","completion":"-1"}} ]}` +// WHAT A SEAT ASKS DECIDES WHICH SCORE IT NEEDS. The three indexes the +// artificial_analysis block publishes describe different work — the agentic and +// coding scores a long tool loop, the intelligence score a single reasoning +// call — so all three travel on the row, and every shape the block can arrive +// in costs only the score it broke. OpenWeights rides the same row: a +// hugging_face_id is the published answer to whether the weights are public. +const benchmarkPayload = `{"data":[ + {"id":"bench/all","name":"All","architecture":{"input_modalities":["text"],"output_modalities":["text"]}, + "benchmarks":{"artificial_analysis":{"intelligence_index":56,"coding_index":41,"agentic_index":33}}, + "hugging_face_id":"deepseek-ai/DeepSeek-V3"}, + {"id":"bench/coding-null","name":"CodingNull","architecture":{"input_modalities":["text"],"output_modalities":["text"]}, + "benchmarks":{"artificial_analysis":{"intelligence_index":40,"coding_index":null,"agentic_index":"high"}}, + "hugging_face_id":""}, + {"id":"bench/no-benchmarks","name":"NoBenchmarks","architecture":{"input_modalities":["text"],"output_modalities":["text"]}} +]}` + +func TestACarriesTheThreeArtificialAnalysisIndexesAndOpenWeights(t *testing.T) { + dir := t.TempDir() + c := Load(context.Background(), Options{ + BaseURL: "https://openrouter.example/api/v1", Dir: dir, + HTTPClient: catalogClient(t, http.StatusOK, benchmarkPayload, nil), + }) + + all, ok := c.Model("bench/all") + if !ok { + t.Fatal("bench/all is missing from the catalog") + } + if all.IntelligenceIndex != 56 || all.CodingIndex != 41 || all.AgenticIndex != 33 { + t.Fatalf("bench/all indexes = %v/%v/%v, want 56/41/33", all.IntelligenceIndex, all.CodingIndex, all.AgenticIndex) + } + if !all.OpenWeights { + t.Fatal("a row with a hugging_face_id read as closed-weights") + } + + // Zero keeps meaning nobody published a score: a null and a non-number are + // absence, exactly like the missing block beside them, and an empty + // hugging_face_id is not a published one. + partial, _ := c.Model("bench/coding-null") + if partial.IntelligenceIndex != 40 || partial.CodingIndex != 0 || partial.AgenticIndex != 0 { + t.Fatalf("bench/coding-null indexes = %v/%v/%v, want 40/0/0", partial.IntelligenceIndex, partial.CodingIndex, partial.AgenticIndex) + } + if partial.OpenWeights { + t.Fatal("an empty hugging_face_id read as open-weights") + } + none, _ := c.Model("bench/no-benchmarks") + if none.IntelligenceIndex != 0 || none.CodingIndex != 0 || none.AgenticIndex != 0 { + t.Fatalf("bench/no-benchmarks invented indexes: %v/%v/%v", none.IntelligenceIndex, none.CodingIndex, none.AgenticIndex) + } + + // A second process reads the file the first one wrote; a cache that dropped + // the scores or the fact would hand it rows the fetch had filled in. + second := Load(context.Background(), Options{ + BaseURL: "https://openrouter.example/api/v1", Dir: dir, + HTTPClient: catalogClient(t, http.StatusInternalServerError, "", nil), + }) + if got, _ := second.Model("bench/all"); got.IntelligenceIndex != 56 || got.CodingIndex != 41 || got.AgenticIndex != 33 || !got.OpenWeights { + t.Fatalf("cached row lost the scores or the fact: %+v", got) + } + + // An older cache predating the fields serves the same rows: absence there + // is unknown, not zero-published, and nothing about loading refuses it. + legacy := t.TempDir() + day := time.Date(2026, 9, 9, 10, 0, 0, 0, time.UTC) + old, err := json.Marshal(cache{FetchedAt: day, Models: []Model{{ID: "legacy/model", IntelligenceIndex: 22}}}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(legacy, cacheName), old, 0o600); err != nil { + t.Fatal(err) + } + fromOld := Load(context.Background(), Options{ + BaseURL: DefaultBaseURL, Dir: legacy, Now: func() time.Time { return day.Add(time.Hour) }, + HTTPClient: failedCatalogClient(), + }) + got, ok := fromOld.Model("legacy/model") + if !ok || got.IntelligenceIndex != 22 || got.CodingIndex != 0 || got.AgenticIndex != 0 { + t.Fatalf("older cache row = %+v found %t", got, ok) + } +} + // WHAT A CACHE READ COSTS IS THE WHOLE OF WHAT A CACHE IS WORTH. A session // re-sends its transcript on every step, so the gap between the prompt price and // the cache-read price is the only number that says what the prefix saved — and diff --git a/internal/catalog/lazy_test.go b/internal/catalog/lazy_test.go index 409d44e7bb..1d7e4efbfb 100644 --- a/internal/catalog/lazy_test.go +++ b/internal/catalog/lazy_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "net/http/httptest" + "sync" "testing" "time" ) @@ -62,6 +63,74 @@ func TestLoadLazyDoesNotWaitForTheFetchAndAnswersTheSame(t *testing.T) { } } +// A warm from a disk cache is a local read, and a caller that waits for it +// within a bound gets the rows rather than an empty answer. +func TestWarmedWaitsForACachedCatalogWithinTheBound(t *testing.T) { + dir := t.TempDir() + if err := Remember(Options{Dir: dir}, []Model{ + {ID: "vendor/seer", Name: "Seer", + InputModalities: []string{"text"}, OutputModalities: []string{"text"}}, + }); err != nil { + t.Fatal(err) + } + lazy := LoadLazy(context.Background(), Options{Dir: dir}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if !lazy.Warmed(ctx) { + t.Fatal("the warm from a fresh disk cache did not land within the bound") + } + if got := lazy.ModelsNow(); len(got) != 1 || got[0].ID != "vendor/seer" { + t.Fatalf("the warmed catalog answers %v, want the cached row", got) + } +} + +// A fetch that never returns never lands, and Warmed says so at the bound — +// the caller's cue to fall back — and then says the opposite once the warm +// finally lands. The bound is the caller's, honoured here and nowhere else. +func TestWarmedAnswersFalseAtTheBoundAndTrueWhenTheWarmLands(t *testing.T) { + release := make(chan struct{}) + var once sync.Once + letGo := func() { once.Do(func() { close(release) }) } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[ + {"id":"vendor/seer","name":"Seer", + "pricing":{"prompt":"0.001","completion":"0.002"}}]}`)) + })) + defer server.Close() + defer letGo() + + lazy := LoadLazy(context.Background(), Options{BaseURL: server.URL, Dir: t.TempDir()}) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + if lazy.Warmed(ctx) { + t.Fatal("a fetch that never returned answered warmed") + } + letGo() + landed, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if !lazy.Warmed(landed) { + t.Fatal("the warm landed and Warmed still said no") + } + if got := lazy.ModelsNow(); len(got) != 1 || got[0].ID != "vendor/seer" { + t.Fatalf("the warmed catalog answers %v, want the fetched row", got) + } +} + +// A catalog that resolved eagerly has nothing to wait for, and no catalog at +// all has nothing to say: both answer at once, in that order. +func TestWarmedAnswersAtOnceForAnEagerCatalogAndNeverForNil(t *testing.T) { + eager := &Catalog{ready: newRows([]Model{{ID: "vendor/seer", OutputModalities: []string{"text"}}})} + if !eager.Warmed(context.Background()) { + t.Fatal("an already resolved catalog answered not warmed") + } + var absent *Catalog + if absent.Warmed(context.Background()) { + t.Fatal("a nil catalog answered warmed") + } +} + // Cleaning dedupes on the literal id, so a slug and its "~" pinned variant can // both survive into the list. A scan answered with the first of them; the index // has to agree. diff --git a/internal/config/auto.go b/internal/config/auto.go new file mode 100644 index 0000000000..4ebc06c447 --- /dev/null +++ b/internal/config/auto.go @@ -0,0 +1,421 @@ +package config + +import ( + "strings" + + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/crewpick" + "github.com/Agent-Field/codeaf/internal/pool/index" +) + +// THE WORD THAT ASKS THE CATALOG TO ANSWER A TIER ROW. +// +// A tier row names a model id, and a model id is a bet the person made one day +// against a catalog that keeps moving under it. `auto` is the row that +// declines to bet: the seat's model is computed from the catalog's own +// published figures — the three capability indexes against the three prices, +// under each seat's call shape (internal/crewpick) — every time the row is +// read, through [autoRow], the one seam both ladders pass through. +// +// NOTHING IS REWRITTEN. The row stays `auto` on disk and the id is derived on +// every read, so a catalog that moves moves the seat with it, and a profile +// never holds a model id this build chose for somebody. + +// AutoValue is the bare word a tier row holds to have its seat computed from +// the catalog. +const AutoValue = "auto" + +// IsAuto reports whether a tier value is the bare word, case folded with the +// surrounding space ignored. `auto:high` is not it: the rows own the +// `:` notation, and a suffixed word is a model id with a level, handed +// on whole the way every other one is. +func IsAuto(value string) bool { + return strings.ToLower(strings.TrimSpace(value)) == AutoValue +} + +// AutoModels is how the catalog reaches seat resolution: the binary holding +// the catalog sets it ONCE AT START-UP, from its non-blocking read, and never +// a fetch — a tier row that says auto resolves from whatever the catalog +// already holds, the same posture every other catalog reader here keeps. Nil, +// and a func answering with no rows, are ordinary states rather than errors: +// a row that says auto then reads the family's table row ([autoRow]), which +// is the answer it had before this word existed. +var AutoModels func() []catalog.Model + +// AutoIndex is how the Model Pool's measurement index reaches seat +// resolution: the binary holding the index sets it ONCE AT START-UP, from +// whatever read it already made, and never a fetch — a tier row that says +// auto resolves from the index already in hand, the same posture AutoModels +// keeps. Nil is an ordinary state, not an error: a row that says auto then +// reads the catalog's own figures alone, which is what it read before this +// seam existed. +var AutoIndex func() *index.Index + +// AutoOwnCells is how an install's own judged scores reach seat resolution: +// the binary holding the pool sets it ONCE AT START-UP, from the own sheet it +// read from disk under the pool directory, the same posture AutoIndex keeps. +// Nil is an ordinary state, not an error: the prior then reads the index's +// cells alone, which is what it read before this seam existed. The cells are +// never held to the index's min_installs — an install's own scores are its +// own evidence, one observation of which is worth having. +var AutoOwnCells func() []crewpick.Cell + +// PoolQualityMetric names the index metric the picker reads a measured seat +// quality from: a gaussian metric whose mean is on the same 0-100 scale +// crewpick scores a seat on, with one cell per role and model. +const PoolQualityMetric = "role_quality" + +// AutoPick is the bare `auto` word's own answer for one tier, computed from +// the rows. It is [AutoPickWith] with the measured quality the Model Pool +// holds carried as a prior — the behaviour the bare word has always had — and +// it stays the seam a row that says `auto` resolves through ([autoRow]). The +// pick row's words answer through [AutoPickWith] directly ([pickedSeat]). +func AutoPick(tier, family, preset string, models []catalog.Model) (modelID string, ok bool) { + return AutoPickWith(tier, family, preset, models, autoPrior(autoIndex())) +} + +// AutoPickWith is the pick itself, with the measured quality NAMED: prior is +// carried into the front as a rating seats read on top of the catalog's own +// published scores, and a nil prior leaves every seat on those scores alone — +// the answer the `catalog` pick word carries, where `learn` passes the pool's +// measurements and the person's own judged runs ([autoPrior]). +// +// It is PURE: no disk, no network, the rows are only read, and the same rows +// give the same answer however often it is asked and in whatever order they +// arrive (crewpick breaks its ties by id, not by order). Each row is read as +// a crewpick candidate — the three indexes, the three prices AS PUBLISHED (a +// uniform scale changes no pick on a front sorted by bill), whether the +// provider published a cache-read price at all, the window, the modalities +// and the parameters — and a row whose price the provider did not publish is +// left out, because a model that may cost anything has no place in a pick +// that is about cost. +// +// tier names the seat the pick is read from: worker, high and mastermind have +// answers, reflex and low and any other word do not. family narrows the +// shelf: `open` picks off the open-weight rows, every other word off the +// whole catalog. preset is one of the three crew words, and any other word +// has no answer. No rows, an empty front or an empty id are no answer too; +// no answer is ok false and an empty id, which is the caller's cue to read +// the family's table row instead ([autoRow], [pickedSeat]). +func AutoPickWith(tier, family, preset string, models []catalog.Model, prior crewpick.Prior) (modelID string, ok bool) { + seat, ok := autoSeat(tier) + if !ok || len(models) == 0 { + return "", false + } + fam := crewpick.All + if normalCrewSource(family) == CrewSourceOpen { + fam = crewpick.Open + } + candidates := autoCandidates(models) + if len(candidates) == 0 { + return "", false + } + frugal, balanced, max := crewpick.Presets(crewpick.FrontWith(candidates, crewpick.DefaultShapes(), fam, prior)) + var pick crewpick.Crew + switch strings.ToLower(strings.TrimSpace(preset)) { + case CrewFrugal: + pick = frugal + case CrewBalanced: + pick = balanced + case CrewMax: + pick = max + default: + return "", false + } + var id string + switch seat { + case crewpick.Worker: + id = pick.Worker + case crewpick.High: + id = pick.High + case crewpick.Mastermind: + id = pick.Mastermind + } + return id, id != "" +} + +// autoIndex is [AutoIndex] read with its ordinary absence folded into one +// answer. +func autoIndex() *index.Index { + if AutoIndex == nil { + return nil + } + return AutoIndex() +} + +// autoPrior reads the measured quality of the index AND of this install's own +// sheet into a crewpick prior. The index's cells of PoolQualityMetric are kept +// only when the metric is declared gaussian — a mean on any other scale would +// be blended against figures it does not share units with — and dropped by +// PriorFromCells below the index's own min_installs, resolved through the +// index's canonical ids so an alias meets its candidate. The own sheet's cells +// are this install's own evidence: they are read through a second +// PriorFromCells at a floor of one and folded into the index's prior seat by +// seat, the means combined by observation count. Their scores are on the +// 0-100 scale a judge answers on whatever the index's metric says, so they +// are read even beside an index whose role_quality is not gaussian. A nil +// index and a nil seam answer no prior, which leaves every seat on the +// catalog quality. +func autoPrior(idx *index.Index) crewpick.Prior { + canonical := autoCanonical(idx) + var indexPrior crewpick.Prior + if idx != nil { + if kind, ok := idx.Kind(PoolQualityMetric); ok && kind == "gaussian" { + cells := idx.Cells(PoolQualityMetric) + measured := make([]crewpick.Cell, 0, len(cells)) + for _, c := range cells { + measured = append(measured, crewpick.Cell{Role: c.Role, Model: c.Model, Mean: c.Mean, N: c.N}) + } + indexPrior = crewpick.PriorFromCells(measured, idx.MinInstalls(), idx.Canonical) + } + } + var ownPrior crewpick.Prior + if own := autoOwnCells(); len(own) > 0 { + ownPrior = crewpick.PriorFromCells(own, 1, canonical) + } + return crewpick.MergePriors(indexPrior, ownPrior) +} + +// autoCanonical is the index's canonical ids, nil when there is no index. The +// own sheet's ids are the ones this install resolved its seats to, and a nil +// canonical leaves them as they stand. +func autoCanonical(idx *index.Index) func(string) string { + if idx == nil { + return nil + } + return idx.Canonical +} + +// autoOwnCells is [AutoOwnCells] read with its ordinary absence folded into +// one answer. +func autoOwnCells() []crewpick.Cell { + if AutoOwnCells == nil { + return nil + } + return AutoOwnCells() +} + +// autoSeat is the tier word that names which of crewpick's three seats the +// pick is read from. The two tiers that never vary — reflex and small work — +// have no answer here, the same law their columns in the shipped tables keep. +func autoSeat(tier string) (crewpick.Seat, bool) { + switch strings.ToLower(strings.TrimSpace(tier)) { + case ModelTierWorker: + return crewpick.Worker, true + case ModelTierHigh: + return crewpick.High, true + case ModelTierMastermind: + return crewpick.Mastermind, true + } + return 0, false +} + +// autoCandidates reads every published row as a candidate. Rows with no +// published price are dropped here, where the reason is nameable, rather than +// being carried into the picker to fall out of its own candidacy law. +func autoCandidates(models []catalog.Model) []crewpick.Candidate { + candidates := make([]crewpick.Candidate, 0, len(models)) + for _, model := range models { + if model.PriceUnknown { + continue + } + candidates = append(candidates, crewpick.Candidate{ + ID: model.ID, + Open: model.OpenWeights, + Intelligence: model.IntelligenceIndex, + Coding: model.CodingIndex, + Agentic: model.AgenticIndex, + PromptPrice: model.PromptPrice, + CompletionPrice: model.CompletionPrice, + CacheReadPrice: model.CacheReadPrice, + HasCacheRead: model.CacheReadPrice > 0, + Context: model.ContextLength, + Images: listHolds(model.InputModalities, "image"), + Tools: listHolds(model.Parameters, "tools"), + }) + } + return candidates +} + +// listHolds says whether a row's own list carries the word, case folded — the +// catalog cleans its lists, but a reader that folds is one fewer assumption. +func listHolds(words []string, word string) bool { + for _, held := range words { + if strings.EqualFold(strings.TrimSpace(held), word) { + return true + } + } + return false +} + +// ── the seam ──────────────────────────────────────────────────────────────── + +// autoRow is THE ONE SEAM both ladders resolve a tier row that says auto +// through — [tierSeatUnder], which the conversation, the settings sheet and +// the crew word read, and [resolveSeat], which every headless door climbs. A +// second copy in each would be a seat that means one thing in chat and +// another headless, which is the defect this file exists to prevent one +// ladder at a time. +// +// The answer is the computed id, on its own rung, when the catalog can +// compute one; otherwise the family's table row FOR THE PRESET, on the table +// rung — the same id a preset write would have landed, and never `auto` and +// never empty. +func autoRow(profileDir, family, tier string) (model string, source SeatSource, preset string) { + preset = crewPresetUnder(profileDir, family) + if id, ok := AutoPick(tier, family, preset, autoCatalogRows()); ok { + return id, SeatComputed, preset + } + if row, ok := CrewModelsForSource(family, preset); ok { + if id := strings.TrimSpace(row[tier]); id != "" { + return id, SeatTable, preset + } + } + // Unreachable for a tier this build knows — every preset row answers all + // five — but the never-empty law is carried rather than assumed: the + // family's default row is the last word. + return defaultTierModel(family, tier), SeatTable, preset +} + +// pickedSeat is THE PICK ROW'S HALF OF THE LADDER: where a seat's model comes +// from when the pick is off the table and the tier row does not name a model +// of its own. Both ladders call it — [tierSeatUnder] for the conversation and +// the settings sheet, [resolveSeat] for every headless door — so a pick +// cannot mean one thing in chat and another headless. +// +// The rule, in the order it is applied: +// +// - the two seats that read every turn are NOT asked: reflex and small work +// always read the table, the same law their columns in the shipped tables +// keep, and the caller decides which tiers reach here; +// - a row holding a model id that is not the preset's own table value is a +// person's own model, and it wins — the pick answers for the seats nobody +// named, not over the names they typed. A row holding the preset's OWN id +// is the preset answering rather than a pin, and the pick computes it; +// - every other seat among worker, careful work and mastermind is computed +// at the crew's preset ([crewPresetUnder]), on the rung the pick names: +// [SeatComputed] under the `catalog` word, [SeatLearned] under `learn`; +// - when the catalog cannot compute one — no rows, no pick off the front — +// the preset's own table row is the answer, on the table rung, the same +// fallback [autoRow] keeps. Never `auto` and never empty. +// +// ok is false only when the pick has nothing to say — the pick row at its +// default, a hand-typed id, or a tier that always reads the table — and the +// caller answers with the row's own rung instead. The DEFAULT PICK IS THE +// FIRST CHECK: a profile that has never answered the row reads every seat +// exactly as it read before the row existed, which is what a default is for. +func pickedSeat(profileDir, family, tier, model string) (Seat, bool) { + if CrewPickAt(profileDir) == CrewPickTable { + return Seat{}, false + } + if tier != ModelTierWorker && tier != ModelTierHigh && tier != ModelTierMastermind { + return Seat{}, false + } + preset := crewPresetUnder(profileDir, family) + if row, ok := CrewModelsForSource(family, preset); ok { + if model != "" && !strings.EqualFold(strings.TrimSpace(model), row[tier]) { + return Seat{}, false + } + } + id, source := pickedModel(tier, family, preset, CrewPickAt(profileDir)) + return Seat{Role: tierSeatRole(tier), Model: id, Source: source, Crew: preset}, true +} + +// pickedModel is the pick word's own answer for one tier: the catalog's pick +// at the preset, on the rung the word names; and, when nothing can be +// computed, the preset's own table row on the table rung — the id a preset +// write would have landed, and never `auto` and never empty. +func pickedModel(tier, family, preset, pick string) (modelID string, source SeatSource) { + pick = normalCrewPick(pick) + var prior crewpick.Prior + if pick == CrewPickLearn { + prior = autoPrior(autoIndex()) + } + if id, ok := AutoPickWith(tier, family, preset, autoCatalogRows(), prior); ok { + if pick == CrewPickLearn { + return id, SeatLearned + } + return id, SeatComputed + } + if row, ok := CrewModelsForSource(family, preset); ok { + if id := strings.TrimSpace(row[tier]); id != "" { + return id, SeatTable + } + } + // Unreachable for a tier this build knows — every preset row answers all + // five — but the never-empty law is carried rather than assumed, the same + // last word [autoRow] keeps. + return defaultTierModel(family, tier), SeatTable +} + +// autoCatalogRows is [AutoModels] read with its two ordinary absences folded +// into one answer. +func autoCatalogRows() []catalog.Model { + if AutoModels == nil { + return nil + } + return AutoModels() +} + +// crewPresetUnder is the preset the profile's five STORED rows make, in the +// family given — read from the rows and never through seat resolution, so +// the seam cannot ask the ladder that is asking it. +// +// Each stored tier value is read through the ladder's own row reader +// ([crewRow]), which is what keeps a row reached through the lineage saying +// it the same way. A row the reader cannot answer — cleared, or never held — +// reads the family's default-preset id, which is what that tier runs until +// somebody writes it. A row that says auto matches WHICHEVER preset is being +// compared, because auto is the one row with no opinion of its own: it is +// asking to be computed at the budget the other four rows name. +// +// The answer is the first preset whose every row matches, the default preset +// tried first and then the crew's own order; when none matches, the default — +// the budget an undecided profile runs at is the budget an auto seat runs at. +// +// THE DEFAULT PRESET WINS EVERY TIE, and ties are ordinary rather than rare: +// the presets differ in only a few cells, so an auto row on a seat the two +// presets share leaves the rest of the rows matching both. Max differs from +// balanced only in the worker seat today, which makes a crew with an auto +// worker and the shipped rows elsewhere exactly that tie — it reads balanced. +// Trying the default first is what decides it, so the order above is the rule +// and not an accident of iteration. +func crewPresetUnder(profileDir, family string) string { + defaults := crewTableFor(family)[DefaultCrew] + read := make(map[string]string, len(ModelTiers)) + for _, tier := range ModelTiers { + value, _, source, cleared := crewRow(profileDir, tier) + switch { + case cleared || source == "": + read[tier] = strings.ToLower(strings.TrimSpace(defaults[tier])) + case IsAuto(value): + read[tier] = AutoValue + default: + read[tier] = strings.ToLower(strings.TrimSpace(value)) + } + } + matches := func(preset string) bool { + row, ok := CrewModelsForSource(family, preset) + if !ok { + return false + } + for _, tier := range ModelTiers { + if read[tier] == AutoValue { + continue + } + if read[tier] != strings.ToLower(strings.TrimSpace(row[tier])) { + return false + } + } + return true + } + if matches(DefaultCrew) { + return DefaultCrew + } + for _, preset := range CrewPresets { + if preset != DefaultCrew && matches(preset) { + return preset + } + } + return DefaultCrew +} diff --git a/internal/config/auto_test.go b/internal/config/auto_test.go new file mode 100644 index 0000000000..0addeb5643 --- /dev/null +++ b/internal/config/auto_test.go @@ -0,0 +1,550 @@ +package config + +import ( + "encoding/json" + "math" + "os" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/crewpick" + "github.com/Agent-Field/codeaf/internal/pool/index" +) + +// THE WORD, AND WHAT IS NOT THE WORD. `auto` is a bare word a tier row holds, +// and everything that only resembles one is a model id: a suffixed word is the +// rows' own `:` notation, a slashed one is a vendor's namespace, and +// `automatic` is somebody's model that happens to begin with it. +func TestIsAutoReadsTheBareWordAlone(t *testing.T) { + for word, want := range map[string]bool{ + "auto": true, " AUTO ": true, "Auto": true, "\tauto\n": true, + "": false, "auto:high": false, "vendor/auto": false, "automatic": false, + } { + if got := IsAuto(word); got != want { + t.Errorf("IsAuto(%q) = %t, want %t", word, got, want) + } + } +} + +// autoTestRows is a small catalog with every fact a pick reads, and one row +// the provider priced "-1" on — which is not a candidate, for the reason a +// pick that is about cost cannot count a model whose cost nobody published. +func autoTestRows() []catalog.Model { + return []catalog.Model{ + {ID: "a/cheap", OpenWeights: true, IntelligenceIndex: 30, CodingIndex: 35, AgenticIndex: 40, + PromptPrice: 0.0000002, CompletionPrice: 0.0000009, CacheReadPrice: 0.00000002, + ContextLength: 200_000, InputModalities: []string{"text", "image"}, Parameters: []string{"tools"}}, + {ID: "a/mid", OpenWeights: true, IntelligenceIndex: 45, CodingIndex: 50, AgenticIndex: 48, + PromptPrice: 0.000001, CompletionPrice: 0.000003, CacheReadPrice: 0.0000001, + ContextLength: 200_000, InputModalities: []string{"text"}, Parameters: []string{"tools"}}, + {ID: "b/care", IntelligenceIndex: 60, CodingIndex: 55, AgenticIndex: 50, + PromptPrice: 0.000004, CompletionPrice: 0.00002, CacheReadPrice: 0.0000004, + ContextLength: 400_000, InputModalities: []string{"text", "image"}, Parameters: []string{"tools"}}, + {ID: "c/mind", IntelligenceIndex: 75, CodingIndex: 70, AgenticIndex: 65, + PromptPrice: 0.00001, CompletionPrice: 0.00005, ContextLength: 200_000, + InputModalities: []string{"text"}, Parameters: []string{"tools"}}, + {ID: "r/router", PriceUnknown: true, IntelligenceIndex: 80, CodingIndex: 80, AgenticIndex: 80, + PromptPrice: -1, CompletionPrice: -1, ContextLength: 200_000}, + } +} + +// THE PICK IS PURE AND ITS ANSWER IS THE FRONT'S. The same rows give the same +// id however often it is asked, the rows are never touched, and a row whose +// price nobody published is in no seat. The front is read the way the shipped +// tables were (crew.go owns the method), so the pick answers exactly what +// crewpick answers for these rows — asserted here against crewpick itself +// rather than against a figure copied out of one. +func TestAutoPickAnswersTheFront(t *testing.T) { + rows := autoTestRows() + workers := make([]string, 0, 3) + for i := 0; i < 3; i++ { + id, ok := AutoPick(ModelTierWorker, CrewSourceAll, CrewFrugal, rows) + if !ok || id == "" { + t.Fatalf("AutoPick answered ok=%t id=%q", ok, id) + } + workers = append(workers, id) + } + for i := 1; i < len(workers); i++ { + if workers[i] != workers[0] { + t.Fatalf("the same rows answered %q then %q — the pick is not pure", workers[0], workers[i]) + } + } + _ = workers + if rows[4].ID != "r/router" || !rows[4].PriceUnknown { + t.Fatal("AutoPick modified the rows it was handed") + } + + frugal, _, maxCrew := crewpick.Presets(crewpick.Front(autoCandidates(rows), crewpick.DefaultShapes(), crewpick.All)) + if len(frugal.Worker) == 0 { + t.Fatal("the test rows field no crew — the rows are broken") + } + if got, _ := AutoPick(ModelTierWorker, CrewSourceAll, CrewFrugal, rows); got != frugal.Worker { + t.Fatalf("worker frugal = %q, want the front's own %q", got, frugal.Worker) + } + if got, _ := AutoPick(ModelTierMastermind, CrewSourceAll, CrewMax, rows); got != maxCrew.Mastermind { + t.Fatalf("mastermind max = %q, want the front's own %q", got, maxCrew.Mastermind) + } +} + +// THE TWO TIERS THAT NEVER VARY HAVE NO ANSWER HERE, the same law their +// columns in the shipped tables keep — reflex and small work are the same +// near-free models in all three presets, and a word that computes for them +// would be a second opinion about a row that has none. +func TestAutoPickRefusesTheTiersWithoutOpinions(t *testing.T) { + for _, tier := range []string{ModelTierReflex, ModelTierLow, "", "banana"} { + if id, ok := AutoPick(tier, CrewSourceAll, CrewFrugal, autoTestRows()); ok || id != "" { + t.Errorf("AutoPick(%q) answered %q, want no answer", tier, id) + } + } +} + +// THE THREE WORDS ARE THE ONLY PRESETS, `open` IS THE ONLY OTHER FAMILY, AND +// NOTHING IS AN ERROR: no answer is ok false and an empty id, on no rows, +// on an empty front, on a word that is not a preset, and on a tier the +// catalog cannot field. +func TestAutoPickAnswersNothingWhenNothingCanBePicked(t *testing.T) { + if id, ok := AutoPick(ModelTierWorker, CrewSourceAll, "banana", autoTestRows()); ok || id != "" { + t.Errorf("an unknown preset answered %q", id) + } + if id, ok := AutoPick(ModelTierWorker, CrewSourceAll, CrewFrugal, nil); ok || id != "" { + t.Errorf("no rows answered %q", id) + } + if id, ok := AutoPick(ModelTierHigh, CrewSourceAll, CrewFrugal, []catalog.Model{{ID: "a/cheap", PromptPrice: 1}}); ok || id != "" { + t.Errorf("rows that field no crew answered %q", id) + } + // A family word this build does not know reads as the default family, + // the way [CrewSourceAt] reads one — and that family answers here. + if _, ok := AutoPick(ModelTierWorker, "misplaced", CrewFrugal, autoTestRows()); !ok { + t.Error("an unknown family word read as no family at all") + } +} + +// ── the ladder ────────────────────────────────────────────────────────────── + +// A ROW THAT SAYS AUTO RESOLVES THROUGH THE ONE SEAM, ON BOTH LADDERS, to the +// computed id with the rung that says so — and with the catalog absent, to +// the family's table row for the preset the other four rows name, on the +// table rung. It never resolves to `auto` and never to empty. +func TestAnAutoRowResolvesOnBothLadders(t *testing.T) { + t.Setenv(ModelEnv, "") + t.Setenv(PlanModelEnv, "") + restore := AutoModels + AutoModels = func() []catalog.Model { return autoTestRows() } + defer func() { AutoModels = restore }() + + rows := writeProfileRows(t, map[string]string{ + KeyTierWorkerModel: " auto ", + KeyTierMastermindModel: "AUTO", + }) + for _, seat := range []Seat{ + TierSeatAt(rows, ModelTierWorker), + ResolveSeats(rows, "", "").Work, + } { + if seat.Source != SeatComputed { + t.Errorf("conversation ladder: the seat reads %s, want computed", seat.Rung()) + } + if strings.TrimSpace(seat.Model) == "" || IsAuto(seat.Model) { + t.Errorf("conversation ladder: the seat reads %q, want a computed id", seat.Model) + } + } + plan := ResolveSeats(rows, "", "").Plan + if plan.Source != SeatComputed { + t.Errorf("headless ladder: the plan seat reads %s, want computed", plan.Rung()) + } + + // The catalog reaches nothing: the family's table row answers, on the + // table rung, at the preset the stored rows make. These rows pin nothing + // else, so they are the default crew — and the worker column of that + // preset in the DEFAULT family is the answer the table owes. + AutoModels = nil + table, _ := CrewModelsForSource(DefaultCrewSource, DefaultCrew) + want := table[ModelTierWorker] + for name, seat := range map[string]Seat{ + "conversation": TierSeatAt(rows, ModelTierWorker), + "headless": ResolveSeats(rows, "", "").Work, + } { + if seat.Model != want || seat.Source != SeatTable { + t.Errorf("%s ladder with no catalog: %q (%s), want %q (table)", name, seat.Model, seat.Rung(), want) + } + } +} + +// A REACHED-THROUGH-THE-LINEAGE ROW SAYS AUTO THE SAME WAY. The worker row +// landed after profiles held only the small-work row, and a profile of that +// vintage whose small-work row says auto hands its work seat the same answer +// the conversation hands it — computed when the catalog can, and never the +// bare word. +func TestAnInheritedAutoRowResolvesTheSameWay(t *testing.T) { + t.Setenv(ModelEnv, "") + t.Setenv(PlanModelEnv, "") + restore := AutoModels + AutoModels = func() []catalog.Model { return autoTestRows() } + defer func() { AutoModels = restore }() + + dir := writeProfileRows(t, map[string]string{KeyTierLowModel: AutoValue}) + for name, seat := range map[string]Seat{ + "conversation": TierSeatAt(dir, ModelTierWorker), + "headless": ResolveSeats(dir, "", "").Work, + } { + if seat.Model == AutoValue || seat.Model == "" { + t.Errorf("%s ladder: the seat reads %q, want what the row was reached through", name, seat.Model) + } + } +} + +// THE FLAG AND THE VARIABLE STILL OUTRANK A ROW THAT SAYS AUTO, and a flag +// whose text is the word is handed on whole — a person who named a model +// called auto got a model called auto, exactly as they did before this rung +// existed. +func TestTheInvocationRungsStillOutrankAnAutoRow(t *testing.T) { + t.Setenv(ModelEnv, "vendor/from-the-environment") + t.Setenv(PlanModelEnv, "vendor/plans-from-the-environment") + dir := writeProfileRows(t, map[string]string{KeyTierWorkerModel: AutoValue}) + seats := ResolveSeats(dir, "", "") + if seats.Work.Model != "vendor/from-the-environment" || seats.Work.Source != SeatEnv { + t.Errorf("the environment rung read %q (%s)", seats.Work.Model, seats.Work.Rung()) + } + if seats.Plan.Model != "vendor/plans-from-the-environment" || seats.Plan.Source != SeatEnv { + t.Errorf("the plan seat read %q (%s)", seats.Plan.Model, seats.Plan.Rung()) + } + seats = ResolveSeats(dir, AutoValue, "") + if seats.Work.Model != AutoValue || seats.Work.Source != SeatFlag { + t.Errorf("a flag that says auto read %q (%s), want the word handed on whole", seats.Work.Model, seats.Work.Rung()) + } +} + +// THE PRESET AN AUTO SEAT RUNS AT IS READ FROM THE STORED ROWS, never through +// seat resolution — the seam cannot ask the ladder that is asking it — and a +// row that says auto matches whichever preset the other rows name. +// +// MAX DIFFERS FROM BALANCED ONLY IN THE WORKER SEAT NOW, so an auto row on the +// worker can no longer tell the two apart — balanced, the default preset, wins +// that tie — and the identifying shape is the other way round: the worker +// pinned to max's own id, which only max names, with the auto row on a seat +// above it. +func TestThePresetAnAutoSeatRunsAtIsReadFromTheStoredRows(t *testing.T) { + t.Setenv(ModelEnv, "") + t.Setenv(PlanModelEnv, "") + models, _ := CrewModelsForSource(DefaultCrewSource, CrewMax) + rows := map[string]string{KeyTierMastermindModel: AutoValue} + for _, tier := range ModelTiers { + if tier != ModelTierMastermind { + rows[tierKeyFor(tier)] = models[tier] + } + } + dir := writeProfileRows(t, rows) + restore := AutoModels + AutoModels = func() []catalog.Model { return autoTestRows() } + seat := TierSeatAt(dir, ModelTierMastermind) + AutoModels = restore + if seat.Crew != CrewMax { + t.Errorf("max's rows and an auto row read as %q", seat.Crew) + } + if seat.Rung() != "crew "+CrewMax+", computed from the catalog" { + t.Errorf("the rung reads %q, want crew max, computed from the catalog", seat.Rung()) + } + // A profile with no auto row anywhere reads as it always read — the + // default rung, no crew word — which is the unchanged-behaviour law: the + // seam fires only on a row that says the word. + seat = TierSeatAt(t.TempDir(), ModelTierWorker) + if seat.Rung() != "default" || seat.Crew != "" { + t.Errorf("an untouched profile reads %q (%s), want the default rung as before", seat.Rung(), seat.Crew) + } +} + +// THE DEFAULT PRESET WINS EVERY TIE. An auto row matches whichever preset is +// being compared, so a profile whose OTHER rows match two presets at once has +// no single answer from the rows alone — and the answer is the default preset, +// balanced, the budget an undecided profile already runs at. +// +// The tie is built here rather than assumed: the worker holds auto and the +// other four rows are taken from max, and the test first states that those +// four are balanced's rows too, which is what makes this a tie at all. Should +// the tables move so the two presets differ somewhere above the worker, that +// first check fails and says so, rather than the test quietly pinning nothing. +func TestTheDefaultPresetWinsWhenTheStoredRowsMatchTwoPresets(t *testing.T) { + t.Setenv(ModelEnv, "") + t.Setenv(PlanModelEnv, "") + balanced, ok := CrewModelsForSource(DefaultCrewSource, CrewBalanced) + if !ok { + t.Fatalf("there is no %s preset in the %s family", CrewBalanced, DefaultCrewSource) + } + max, ok := CrewModelsForSource(DefaultCrewSource, CrewMax) + if !ok { + t.Fatalf("there is no %s preset in the %s family", CrewMax, DefaultCrewSource) + } + rows := map[string]string{KeyTierWorkerModel: AutoValue} + for _, tier := range ModelTiers { + if tier == ModelTierWorker { + continue + } + if balanced[tier] != max[tier] { + t.Fatalf("the %s seat differs between balanced (%s) and max (%s), so an auto worker is no longer a tie", + tier, balanced[tier], max[tier]) + } + rows[tierKeyFor(tier)] = max[tier] + } + if balanced[ModelTierWorker] == max[ModelTierWorker] { + t.Fatalf("balanced and max name the same worker, so the two presets are not two") + } + + dir := writeProfileRows(t, rows) + restore := AutoModels + AutoModels = func() []catalog.Model { return autoTestRows() } + seat := TierSeatAt(dir, ModelTierWorker) + AutoModels = restore + + if seat.Crew != CrewBalanced { + t.Errorf("rows matching both presets read as %q, want %s, the default preset", seat.Crew, CrewBalanced) + } + if seat.Rung() != "crew "+CrewBalanced+", computed from the catalog" { + t.Errorf("the rung reads %q, want crew %s, computed from the catalog", seat.Rung(), CrewBalanced) + } + // And the same tie on the table rung, with nothing to compute from: the + // default preset decides the fallback id too, not just the budget word. + dir = writeProfileRows(t, rows) + seat = TierSeatAt(dir, ModelTierWorker) + if seat.Crew != CrewBalanced || seat.Source != SeatTable { + t.Errorf("with no catalog the tie reads %q (%s), want balanced on the table rung", seat.Crew, seat.Rung()) + } + if seat.Model != balanced[ModelTierWorker] { + t.Errorf("the tie's table id is %q, want balanced's own worker %q", seat.Model, balanced[ModelTierWorker]) + } +} + +// A SETTINGS ROW SHOWS THE MODEL RUNNING, because it reads through the +// resolver — and a profile that says auto nowhere is untouched by any of +// this: the same rows, the same ids, the same rungs. +func TestAutoNowhereChangesNothing(t *testing.T) { + t.Setenv(ModelEnv, "") + t.Setenv(PlanModelEnv, "") + restore := AutoModels + AutoModels = func() []catalog.Model { return autoTestRows() } + defer func() { AutoModels = restore }() + + dir := t.TempDir() + if err := ApplyCrew(dir, CrewBalanced); err != nil { + t.Fatal(err) + } + before := map[string]string{} + for _, tier := range ModelTiers { + before[tier] = TierModelAt(dir, tier) + } + raw, err := os.ReadFile(BudgetConfigPath(dir)) + if err != nil { + t.Fatal(err) + } + var stored map[string]any + if err := json.Unmarshal(raw, &stored); err != nil { + t.Fatal(err) + } + if len(stored) == 0 { + t.Fatal("the write stored nothing") + } + for tier, want := range before { + if got := TierModelAt(dir, tier); got != want { + t.Errorf("%s read %q then %q — a read rewrote a row", tier, want, got) + } + } + if got := CrewAt(dir); got != CrewBalanced { + t.Errorf("the crew word reads %q, want balanced", got) + } +} + +// THE SEAM'S TABLE RUNG IS THE PRESET'S OWN ID, not merely a non-empty +// answer: with the catalog gone, an auto row under rows that pin max reads +// to that preset's id for the tier, not to the default preset's. The worker +// is the pinned seat — it is the one cell where max differs from balanced — +// and the auto row rides the mastermind. +func TestTheTableRungAnswersThePresetTheRowsName(t *testing.T) { + t.Setenv(ModelEnv, "") + t.Setenv(PlanModelEnv, "") + models, _ := CrewModelsForSource(DefaultCrewSource, CrewMax) + rows := map[string]string{KeyTierMastermindModel: AutoValue} + for _, tier := range ModelTiers { + if tier != ModelTierMastermind { + rows[tierKeyFor(tier)] = models[tier] + } + } + dir := writeProfileRows(t, rows) + seat := TierSeatAt(dir, ModelTierMastermind) + if seat.Model != models[ModelTierMastermind] || seat.Source != SeatTable { + t.Errorf("with no catalog the seat reads %q (%s), want max's own mastermind id on the table rung", + seat.Model, seat.Rung()) + } +} + +// mustIndex parses a measurement document or fails the test. +func mustIndex(t *testing.T, doc string) *index.Index { + t.Helper() + x, err := index.Parse([]byte(doc)) + if err != nil { + t.Fatalf("parse index: %v", err) + } + return x +} + +// priorDocument is the smallest index document that rates one catalog row on +// the worker seat: a gaussian role_quality metric and a cell well above the +// document's min_installs. +const priorDocument = `{ + "schema": 1, + "generated": "2026-09-17", + "min_installs": 5, + "metrics": {"role_quality": {"kind": "gaussian", "dims": ["role", "model"]}}, + "cells": [ + {"metric": "role_quality", "role": "worker", "model": "a/cheap", "mean": 95, "sd": 5, "n": 1000} + ] +}` + +// THE INDEX REACHES THE PICK THROUGH THE PRIOR. An index whose role_quality +// metric rates one catalog row well above its published worker quality moves +// AutoPick's worker answer to that row, and with no index the pick answers +// exactly what the front answers, as before. +func TestAutoPickReadsAMeasuredQualityPrior(t *testing.T) { + restore := AutoIndex + defer func() { AutoIndex = restore }() + + rows := autoTestRows() + + AutoIndex = nil + frugal, _, _ := crewpick.Presets(crewpick.Front(autoCandidates(rows), crewpick.DefaultShapes(), crewpick.All)) + before, ok := AutoPick(ModelTierWorker, CrewSourceAll, CrewFrugal, rows) + if !ok || before != frugal.Worker { + t.Fatalf("with no index the worker pick reads %q, want the front's own %q", before, frugal.Worker) + } + + AutoIndex = func() *index.Index { return mustIndex(t, priorDocument) } + after, ok := AutoPick(ModelTierWorker, CrewSourceAll, CrewFrugal, rows) + if !ok { + t.Fatal("the pick answers nothing with the index in hand") + } + if after == before { + t.Fatalf("the rated row left the worker pick at %q; the index never reached the quality", after) + } + if after != "a/cheap" { + t.Fatalf("the rated row did not take the worker seat: got %q", after) + } +} + +// THE PRIOR IS THE ARGUMENT, AND NOTHING HIDES IT. AutoPickWith carries +// whatever prior it is given: nil leaves the worker on the front's own +// answer, and the same measured cell the AutoPick test reads moves it — which +// is exactly the difference between the `catalog` and `learn` pick words +// (crew.go's [CrewPickAt]). +func TestAutoPickWithCarriesThePriorItIsGiven(t *testing.T) { + rows := autoTestRows() + front, _, _ := crewpick.Presets(crewpick.Front(autoCandidates(rows), crewpick.DefaultShapes(), crewpick.All)) + bare, ok := AutoPickWith(ModelTierWorker, CrewSourceAll, CrewFrugal, rows, nil) + if !ok || bare != front.Worker { + t.Fatalf("with no prior the worker pick reads %q, want the front's own %q", bare, front.Worker) + } + measured, ok := AutoPickWith(ModelTierWorker, CrewSourceAll, CrewFrugal, rows, autoPrior(mustIndex(t, priorDocument))) + if !ok { + t.Fatal("the pick answers nothing with the prior in hand") + } + if measured == bare { + t.Fatalf("the measured cell left the worker pick at %q; the prior never reached the front", measured) + } + if measured != "a/cheap" { + t.Fatalf("the rated row did not take the worker seat: got %q", measured) + } +} + +// ── the install's own sheet ───────────────────────────────────────────────── + +// ownCells is the install's own evidence for the prior tests: one cell the +// index document also carries, one it does not, both on the worker seat. +func ownCells() []crewpick.Cell { + return []crewpick.Cell{ + {Role: "worker", Model: "a/cheap", Mean: 90, N: 3}, + {Role: "worker", Model: "b/only", Mean: 40, N: 1}, + } +} + +// With no index in hand the own sheet's cells are the whole prior — and a +// one-observation cell of this install's own is worth having, where an +// index's min_installs would have dropped it. +func TestAutoPriorReadsTheOwnCellsAloneWithNoIndex(t *testing.T) { + restore := AutoOwnCells + defer func() { AutoOwnCells = restore }() + AutoOwnCells = ownCells + + prior := autoPrior(nil) + if len(prior[crewpick.Worker]) != 2 { + t.Fatalf("the own cells made a prior of %d worker ratings, want two: %v", len(prior[crewpick.Worker]), prior) + } + if r := prior[crewpick.Worker]["b/only"]; r.Mean != 40 || r.N != 1 { + t.Fatalf("a one-observation own cell did not survive: %v", r) + } + if r := prior[crewpick.Worker]["a/cheap"]; r.Mean != 90 || r.N != 3 { + t.Fatalf("the shared cell came back %v, want mean 90 over 3", r) + } +} + +// A model both the index and the own sheet rate folds by observation count: +// the counts add, the mean is the mean of the means weighted by them, and the +// cells either one holds alone are carried beside it. +func TestAutoPriorFoldsTheOwnCellsIntoTheIndexByObservationCount(t *testing.T) { + restoreIndex, restoreOwn := AutoIndex, AutoOwnCells + defer func() { AutoIndex, AutoOwnCells = restoreIndex, restoreOwn }() + AutoIndex = func() *index.Index { return mustIndex(t, priorDocument) } + AutoOwnCells = ownCells + + prior := autoPrior(AutoIndex()) + shared := prior[crewpick.Worker]["a/cheap"] + wantMean := (1000*95.0 + 3*90.0) / 1003 + if shared.N != 1003 { + t.Fatalf("the shared cell folded to N %d, want 1003", shared.N) + } + if math.Abs(shared.Mean-wantMean) > 1e-9 { + t.Fatalf("the shared cell folded to mean %v, want %v", shared.Mean, wantMean) + } + if r := prior[crewpick.Worker]["b/only"]; r.Mean != 40 || r.N != 1 { + t.Fatalf("the own-only cell did not survive the fold: %v", r) + } +} + +// A nil seam changes nothing: the index's prior is exactly what it was, and +// with no index there is no prior at all. +func TestANilOwnCellsSeamLeavesThePriorAlone(t *testing.T) { + restoreIndex, restoreOwn := AutoIndex, AutoOwnCells + defer func() { AutoIndex, AutoOwnCells = restoreIndex, restoreOwn }() + AutoIndex = func() *index.Index { return mustIndex(t, priorDocument) } + AutoOwnCells = nil + + prior := autoPrior(AutoIndex()) + if len(prior[crewpick.Worker]) != 1 || prior[crewpick.Worker]["a/cheap"] != (crewpick.Rating{Mean: 95, N: 1000}) { + t.Fatalf("a nil seam changed the index prior: %v", prior) + } + if prior := autoPrior(nil); prior != nil { + t.Fatalf("a nil seam and no index made a prior: %v", prior) + } +} + +// The own sheet's scores are on the 0-100 scale a judge answers on whatever +// the index's metric says, so they are read even beside an index whose +// role_quality is not gaussian — while the index's own cells are not. +func TestAutoPriorReadsTheOwnCellsBesideANonGaussianIndex(t *testing.T) { + restoreIndex, restoreOwn := AutoIndex, AutoOwnCells + defer func() { AutoIndex, AutoOwnCells = restoreIndex, restoreOwn }() + AutoIndex = func() *index.Index { + return mustIndex(t, `{ + "schema": 1, + "generated": "2026-09-17", + "min_installs": 1, + "metrics": {"role_quality": {"kind": "tally", "dims": ["role", "model"]}}, + "cells": [{"metric": "role_quality", "role": "worker", "model": "a/cheap", "mean": 5, "n": 900}] + }`) + } + AutoOwnCells = ownCells + + prior := autoPrior(AutoIndex()) + if r, ok := prior[crewpick.Worker]["a/cheap"]; !ok || r.Mean != 90 || r.N != 3 { + t.Fatalf("the own cell was not read beside a non-gaussian index: %v ok %v", prior[crewpick.Worker]["a/cheap"], ok) + } + if _, ok := prior[crewpick.Worker]["b/only"]; !ok { + t.Fatalf("the second own cell did not survive: %v", prior) + } +} diff --git a/internal/config/crew.go b/internal/config/crew.go index 0e9ed04c9b..4380879ea1 100644 --- a/internal/config/crew.go +++ b/internal/config/crew.go @@ -22,17 +22,14 @@ import ( // asked to think. // // TWO FAMILIES BEHIND THE SAME THREE WORDS. Every preset exists twice: once in -// the open family, every seat an open-weight model, which is the property that -// makes a shipped default defensible, because nobody's default crew should be -// a bet on one vendor's pricing; and once in the all family, the same three -// words resolved over the whole catalog with closed and frontier models in it, -// for the person who has chosen to spend what those cost. Which family the -// words draw from is one row, [KeyCrewSource], and open is what it reads when -// nobody has answered it: the all family is a thing a person asks for, never -// one they get by saying nothing. Both the derived reading and the write -// resolve through the row ([CrewSourceAt]), so the word on the sheet can never -// mean one family while the five values it summarizes were drawn from the -// other. +// the all family, the same three words resolved over the whole catalog with +// closed and frontier models in it; and once in the open family, every seat an +// open-weight model, for the person who wants the three words to stay on that +// shelf whatever the catalog does around it. Which family the words draw from +// is one row, [KeyCrewSource], and all is what it reads when nobody has +// answered it. Both the derived reading and the write resolve through the row +// ([CrewSourceAt]), so the word on the sheet can never mean one family while +// the five values it summarizes were drawn from the other. // // THE PRESET IS DERIVED AND NEVER STORED. [CrewAt] reads the five live tier // values and answers which preset they are, or "custom". A stored word would be @@ -63,121 +60,259 @@ const ( var CrewPresets = []string{CrewFrugal, CrewBalanced, CrewMax} // DefaultCrew is what a profile nobody has touched reads. It is balanced because -// the five shipped tier defaults ARE the balanced row — see [crewModels] — and -// that identity is asserted by a test rather than trusted. +// the five shipped tier defaults ARE the balanced row of the DEFAULT FAMILY — +// see [crewAllModels] — and that identity is asserted by a test rather than +// trusted. const DefaultCrew = CrewBalanced // The two families the preset words can draw from. They are the values // [KeyCrewSource] takes, spelled here once and read by the row, the resolver // and the manual. const ( - // CrewSourceOpen is the open-weight family and the shipped default. + // CrewSourceOpen is the open-weight family. CrewSourceOpen = "open" - // CrewSourceAll is the whole catalog, closed and frontier models included. + // CrewSourceAll is the whole catalog, closed and frontier models included, + // and the family a profile that has answered nothing resolves. CrewSourceAll = "all" ) -// CrewSources lists them, open first, which is also the default and the order -// the row widens in. +// CrewSources lists them, open first, which is the order the row widens in: the +// narrower shelf, then the whole catalog. The default is the second of them, +// [DefaultCrewSource], because this list is about width and not about which one +// a profile starts on. var CrewSources = []string{CrewSourceOpen, CrewSourceAll} -// DefaultCrewSource is open, for the same reason the shipped crew is open -// weights: a family nobody chose is the one this build can defend. -const DefaultCrewSource = CrewSourceOpen +// DefaultCrewSource is all: the three words are read off the whole catalog +// unless the row says otherwise, and the five shipped tier defaults are that +// family's balanced row. +const DefaultCrewSource = CrewSourceAll -// crewModels is the whole table: one row per preset, one model per class. +// ── where the seats are picked from ──────────────────────────────────────── + +// THE THIRD ROW THE CREW WORDS ARE ANSWERED THROUGH. The crew row says how +// much to spend and the family row says which shelf those budgets name; the +// pick row says where the models for that money come from when a tier row +// does not hold a model id of its own: +// +// - `table` — the rows this build measured and shipped ([crewModels] and +// [crewAllModels]), which is what an unwritten seat has always read; +// - `catalog` — the same three budgets recomputed off the catalog's own +// published prices and scores, on every read, with no measurement of +// anybody's own runs in it ([AutoPickWith] with no prior); +// - `learn` — the catalog computation plus the Model Pool's measurements +// and the person's own judged runs, carried as a quality prior +// ([autoPrior]). +// +// THE DEFAULT IS THE TABLE because the table is what a profile has always +// read: an unwritten seat names the preset's own row, and nothing about a +// profile that has answered nothing moves until somebody answers a row. The +// other two words are an opt-in to a read that keeps moving — a seat that +// follows the catalog follows it whether or not the shipped rows do — and a +// person has to say so. +// +// A PICK NEVER OVERRIDES A MODEL ID. The row answers for the seats a person +// did not name, and the seats they did — written by hand, or by a preset — +// keep their ids until the crew is picked again, except that a row holding +// the preset's own table value is the preset answering, not a person pinning +// one model by id. The rule is [pickedSeat]'s to apply and the manual's to +// state. +const ( + // CrewPickTable is the measured rows this build ships, and the default. + CrewPickTable = "table" + // CrewPickCatalog is the catalog's own published figures, with nothing + // measured on top. + CrewPickCatalog = "catalog" + // CrewPickLearn is the catalog computation plus the Model Pool's + // measurements and the person's own judged runs. + CrewPickLearn = "learn" +) + +// CrewPicks lists the words a person may WRITE, narrowest first: the shipped +// table, then the catalog on its own, then the catalog with what runs +// measured. A pick word is not a preset and names no budget — the crew row +// above it still does that. +var CrewPicks = []string{CrewPickTable, CrewPickCatalog, CrewPickLearn} + +// DefaultCrewPick is the table: the rows this build measured are where an +// unwritten seat's model comes from until somebody answers the row. +const DefaultCrewPick = CrewPickTable + +// knownCrewPick folds a word and says whether it is one of the picks this +// build knows. It is the ONE place the fold is spelled: [SetCrewPick] and the +// ladder's [pickedSeat] both go through it, so a pick added to [CrewPicks] is +// accepted everywhere at once. A reader folds a word it does not know to the +// default pick; a writer refuses it. +func knownCrewPick(pick string) (string, bool) { + pick = strings.ToLower(strings.TrimSpace(pick)) + for _, known := range CrewPicks { + if pick == known { + return known, true + } + } + return "", false +} + +// normalCrewPick folds a pick word to one of the three this build knows, +// reading a word it does not know as the default pick. +func normalCrewPick(pick string) string { + if known, ok := knownCrewPick(pick); ok { + return known + } + return DefaultCrewPick +} + +// CrewPickAt is where the seats are picked from on this profile, +// [DefaultCrewPick] when the row is absent. A word this build does not know +// reads as the default pick, silently, the way a retired choice reads +// everywhere else on this sheet. +func CrewPickAt(profileDir string) string { + if value, ok := persistedString(profileDir, KeyCrewPick); ok { + return normalCrewPick(value) + } + return DefaultCrewPick +} + +// AnyTierAutoAt answers whether any of the five tier rows reads `auto` on this +// profile — written directly, or reaching the word through an older row +// ([crewRow]) — which is the second way a seat resolution is computed from the +// catalog's rows, beside the pick row ([CrewPickAt]). A door that resolves +// seats asks both before it resolves, because both answers are computed from +// the rows the process already holds, and a door that asks before they land +// reads the family's table row over a profile that never chose it. +func AnyTierAutoAt(profileDir string) bool { + for _, tier := range ModelTiers { + if model, _, _, _ := crewRow(profileDir, tier); IsAuto(model) { + return true + } + } + return false +} + +// SetCrewPick writes the pick row ALONE, in one file write. The word is +// refused the way every choice row refuses one, so a typo cannot land a pick +// nothing reads. It writes no tier row: the pick says where seats are read +// from, and the seats keep the ids on disk until the crew is picked again. +func SetCrewPick(profileDir, pick string) error { + known, ok := knownCrewPick(pick) + if !ok { + return fmt.Errorf("pick one of: %s", strings.Join(CrewPicks, ", ")) + } + return writeProfileValue(profileDir, KeyCrewPick, known) +} + +// crewModels is the open-weight table: one row per preset, one model per class. +// +// THE WORKER COLUMN IS THE DIAL. It holds glm-5.3-flash through balanced, and +// max is the preset that takes it to glm-5.3, because it is the seat that pays +// most of a task's bill, and a preset that moved every other seat while leaving +// it alone would change everything about a task except its cost. The careful +// column always sees images (the vision role rides it) and is a second vendor +// from balanced upward; frugal keeps worker and careful on the same +// glm-5.3-flash, because at that bill the open-weight front has no second +// vendor to take the careful seat. The reflex and low columns never vary: they +// are the same near-free models in all three presets, and a column that never +// varies is not a dial. // -// THE WORKER COLUMN IS THE DIAL. It climbs the open-weight pareto front one step -// per preset — deepseek-v4-flash, glm-5.3-flash, glm-5.3 — because it is the -// seat that pays most of a task's bill, and a preset that moved every other seat -// while leaving it alone would change everything about a task except its cost. -// The mastermind column buys a bigger planning model without choosing its -// generation behavior. The careful column is always a DIFFERENT VENDOR from -// the worker and always sees images (the vision role rides it). The reflex and -// low columns never vary: they are the same -// near-free models in all three presets, and a column that never varies is not -// a dial. +// HOW THE IDS WERE READ OFF, on 2026-09-16 and seat by seat. Every open-weight +// row of the catalog was placed on two axes: the expected bill that seat's own +// call shape runs up, built from the catalog's published prompt, completion and +// cache-read prices, against that seat's quality, taken from its published +// intelligence, coding and agentic indexes. THE CALL SHAPE IS PART OF THE +// PRICE. The worker and the careful seats were costed as LONG CACHED LOOPS — a +// large prompt read back turn after turn, so the cache-read price carries most +// of the weight — and the mastermind as ONE-SHOT CALLS, where the prompt is +// paid in full each time and there are few of them. Each preset then takes, for +// each seat, a point on the pareto front of that plot at the bill it is willing +// to run: nothing on the front costs less at the same quality, and nothing at +// the same bill scores higher. // -// Every id is an open-weight model, picked off the catalog's own published -// scores against blended price: 2026-09-01 for the roster as a whole, with the -// frugal mastermind and the balanced careful and mastermind re-picked on -// 2026-09-16 (the manual's open table lists every seat; settings.go's -// DefaultWorkerModel says how). No closed model is here: the open family is the -// shelf that has to stand on price alone, and the `all` family is where a closed -// model goes. +// That is why the columns do not climb together. Under one-shot pricing +// glm-5.3-flash is on the front at frugal's bill and glm-5.3 is the next point +// above it, so the mastermind column reads flash, glm-5.3, glm-5.3; under +// long-loop pricing the same plot puts kimi-k3 on the careful seat from +// balanced upward, which is also the second vendor that seat has to be. +// +// No closed model is here: the open family is the shelf that stands on the +// open-weight rows alone, and the `all` family is where a closed model goes. var crewModels = map[string]map[string]string{ CrewFrugal: { ModelTierReflex: "mistralai/mistral-nemo", ModelTierLow: "deepseek/deepseek-v4-flash-0731", - ModelTierWorker: "deepseek/deepseek-v4-flash-0731", + ModelTierWorker: "z-ai/glm-5.3-flash", ModelTierHigh: "z-ai/glm-5.3-flash", - ModelTierMastermind: "z-ai/glm-5.3", + ModelTierMastermind: "z-ai/glm-5.3-flash", }, CrewBalanced: { ModelTierReflex: "mistralai/mistral-nemo", ModelTierLow: "deepseek/deepseek-v4-flash-0731", ModelTierWorker: "z-ai/glm-5.3-flash", ModelTierHigh: "moonshotai/kimi-k3", - ModelTierMastermind: "moonshotai/kimi-k3", + ModelTierMastermind: "z-ai/glm-5.3", }, CrewMax: { ModelTierReflex: "mistralai/mistral-nemo", ModelTierLow: "deepseek/deepseek-v4-flash-0731", ModelTierWorker: "z-ai/glm-5.3", ModelTierHigh: "moonshotai/kimi-k3", - ModelTierMastermind: "moonshotai/kimi-k3", + ModelTierMastermind: "z-ai/glm-5.3", }, } // crewAllModels is the same three presets answered from the whole catalog // rather than its open-weight shelf, which is what the `all` family of -// [KeyCrewSource] draws from. The careful column is the same law here as -// there: a DIFFERENT VENDOR from the worker in every preset, and the reflex -// and low columns still never vary. The ids are locked the way the open ones -// are, off the catalog's own published scores against blended price, and -// closed models live here and only here. +// [KeyCrewSource] draws from AND WHAT A PROFILE THAT HAS ANSWERED NOTHING +// RESOLVES ([DefaultCrewSource]). The careful column is a DIFFERENT VENDOR +// from the worker in every preset here — the open family's frugal row is the +// one standing exception — and the reflex and low columns still never vary. +// Closed models live here and only here. +// +// The ids come off the same plot [crewModels] describes, run on 2026-09-16 over +// every row of the catalog rather than the open-weight ones: expected task bill +// against seat quality, the bill built from the published prompt, completion +// and cache-read prices under each seat's own call shape — the worker and the +// careful seats as long cached loops, the mastermind as one-shot calls — and +// the quality from the published intelligence, coding and agentic indexes. // -// TWO COLUMNS SPEND DIFFERENTLY HERE. The worker column is [crewModels]'s dial -// but moves once, not per preset: gpt-5.6-sol holds frugal and balanced, and -// only max buys claude-fable-5.1, because on the catalog the coding ceiling is -// the expensive step and the two steps below it are the same model. What -// frugal-to-balanced buys here is the careful seat (gemini-3.8-flash to -// claude-opus-5) and a smarter mastermind, not a dearer worker. And max's -// mastermind sits on the worker's own model on purpose: fable-5.1 is the -// catalog's ceiling, so there is no better planner to buy above it. +// THE WORKER STAYS ON glm-5.3-flash THROUGH BALANCED, and that is the whole +// shape of this table. The worker seat carries most of a task's tokens, so a +// step there multiplies through the entire bill while a step on the careful or +// the mastermind seat is paid a handful of times. The money therefore goes to +// the two low-volume seats first: frugal to balanced moves the careful seat to +// claude-fable-5.1 and the mastermind to claude-opus-5, and max moves the +// worker itself, with the careful seat staying on fable beside it. var crewAllModels = map[string]map[string]string{ CrewFrugal: { ModelTierReflex: "google/gemini-2.5-flash", ModelTierLow: "deepseek/deepseek-v4-flash-0731", - ModelTierWorker: "openai/gpt-5.6-sol", - ModelTierHigh: "google/gemini-3.8-flash", - ModelTierMastermind: "anthropic/claude-opus-5", + ModelTierWorker: "z-ai/glm-5.3-flash", + ModelTierHigh: "qwen/qwen3.8-max-0902", + ModelTierMastermind: "z-ai/glm-5.3-flash", }, CrewBalanced: { ModelTierReflex: "google/gemini-2.5-flash", ModelTierLow: "deepseek/deepseek-v4-flash-0731", - ModelTierWorker: "openai/gpt-5.6-sol", - ModelTierHigh: "anthropic/claude-opus-5", - ModelTierMastermind: "anthropic/claude-fable-5.1", + ModelTierWorker: "z-ai/glm-5.3-flash", + ModelTierHigh: "anthropic/claude-fable-5.1", + ModelTierMastermind: "anthropic/claude-opus-5", }, CrewMax: { ModelTierReflex: "google/gemini-2.5-flash", ModelTierLow: "deepseek/deepseek-v4-flash-0731", - ModelTierWorker: "anthropic/claude-fable-5.1", - ModelTierHigh: "openai/gpt-6-astra", - ModelTierMastermind: "anthropic/claude-fable-5.1", + ModelTierWorker: "z-ai/glm-5.3", + ModelTierHigh: "anthropic/claude-fable-5.1", + ModelTierMastermind: "anthropic/claude-opus-5", }, } -// crewTableFor is the family one source word names. It is total: `all` names -// the catalog-wide family, and every other reading (blank, misspelt, a word a -// later build retired) names the open one, because a family nobody asked for -// is the one this build can defend. +// crewTableFor is the family one source word names. It is total: `open` names +// the open-weight family, and every other reading (blank, misspelt, a word a +// later build retired) names the catalog-wide one, because an unreadable answer +// resolves the same family an absent answer does ([DefaultCrewSource]). func crewTableFor(source string) map[string]map[string]string { - if normalCrewSource(source) == CrewSourceAll { - return crewAllModels + if normalCrewSource(source) == CrewSourceOpen { + return crewModels } - return crewModels + return crewAllModels } // knownCrewSource folds a word and says whether it is one of the families this @@ -206,8 +341,8 @@ func normalCrewSource(source string) string { // CrewSourceAt is which family the preset words draw from on this profile, // [DefaultCrewSource] when the row is absent. A word this build does not know -// reads as open, silently, the way a retired choice reads everywhere else on -// this sheet. +// reads as the default family, silently, the way a retired choice reads +// everywhere else on this sheet. func CrewSourceAt(profileDir string) string { if value, ok := persistedString(profileDir, KeyCrewSource); ok { return normalCrewSource(value) @@ -237,37 +372,37 @@ func CrewModelsForSource(source, preset string) (map[string]string, bool) { // contradiction the chooser exists to prevent. var crewLines = map[string]map[string]string{ CrewSourceOpen: { - CrewFrugal: "deepseek works, glm-5.3 thinks · pennies a day", - CrewBalanced: "glm-flash works, kimi-k3 checks and thinks", - CrewMax: "glm-5.3 works, kimi-k3 thinks and checks", + CrewFrugal: "glm-flash works, checks and thinks · pennies a day", + CrewBalanced: "glm-flash works, kimi-k3 checks, glm-5.3 thinks", + CrewMax: "glm-5.3 works and thinks, kimi-k3 checks", }, CrewSourceAll: { - CrewFrugal: "gpt-5.6-sol works, gemini-flash checks, opus thinks", - CrewBalanced: "gpt-5.6-sol works, opus checks, fable thinks", - CrewMax: "fable works and thinks, astra checks", + CrewFrugal: "glm-flash works and thinks, qwen-max checks", + CrewBalanced: "glm-flash works, fable checks, opus thinks", + CrewMax: "glm-5.3 works, fable checks, opus thinks", }, } -// CrewLine is one preset's own line in the OPEN family, empty for a word that is -// not a preset. The family-aware spelling is [CrewLineFor]. +// CrewLine is one preset's own line in the DEFAULT family, empty for a word +// that is not a preset. The family-aware spelling is [CrewLineFor]. func CrewLine(preset string) string { - return CrewLineFor(CrewSourceOpen, preset) + return CrewLineFor(DefaultCrewSource, preset) } // CrewLineFor is one preset's own line in one family, empty for a word that is -// not a preset. A family this build does not know reads as open, the way the -// row does. +// not a preset. A family this build does not know reads as the default one, the +// way the row does. func CrewLineFor(source, preset string) string { return crewLines[normalCrewSource(source)][strings.ToLower(strings.TrimSpace(preset))] } -// CrewModels is the five models one preset would set in the OPEN family, by +// CrewModels is the five models one preset would set in the DEFAULT family, by // tier word: the family a profile nobody has touched reads, and the spelling // the callers hold. The family-aware spelling is // [CrewModelsForSource]. It returns a copy, because a caller printing the // table must not be able to edit it. func CrewModels(preset string) (map[string]string, bool) { - return CrewModelsForSource(CrewSourceOpen, preset) + return CrewModelsForSource(DefaultCrewSource, preset) } // CrewAt is the crew as the five live tier values make it: the preset they are, @@ -282,10 +417,19 @@ func CrewModels(preset string) (map[string]string, bool) { // // The comparison runs against the family [CrewSourceAt] names, so the reading // moves with the row and never behind it: flip the family and a crew the old -// family wrote matches nothing, which reads as custom and is true, because -// five open ids are not any all-family preset either. +// family wrote matches nothing, which reads as custom and is true, because one +// family's five ids are not any preset of the other. func CrewAt(profileDir string) string { family := CrewSourceAt(profileDir) + if CrewPickAt(profileDir) != CrewPickTable { + // WITH THE PICK OFF THE TABLE the three dial seats are computed ids the + // preset tables do not hold, and comparing the live seats would read + // custom over a crew the person chose. The word answers what the STORED + // rows make instead — the budget the seats are computed at ([pickedSeat] + // reads the same rows) — so the word on the sheet stays the decision it + // summarizes while the ids underneath move with the catalog. + return crewStoredAt(profileDir, family) + } live := make(map[string]string, len(ModelTiers)) for _, tier := range ModelTiers { live[tier] = tierSeatUnder(profileDir, family, tier).Model @@ -299,6 +443,63 @@ func CrewAt(profileDir string) string { return CrewCustom } +// crewStoredAt is the crew the STORED five rows make, in the family given: +// the preset they are, or [CrewCustom]. It is [CrewAt]'s reading when the +// pick row takes the seats off the table, where the live comparison would +// compare computed ids. +// +// Each row is read through the ladder's own row reader ([crewRow]). A row the +// reader cannot answer because the key was NEVER HELD reads the family's +// default-preset id, which is what that tier runs until somebody writes it; +// a row CLEARED ON PURPOSE reads empty and matches nothing, because +// "follows the conversation" is not any of the three; a row that says auto is +// skipped, because auto is the one row with no opinion of its own — it runs +// at whatever budget the rows around it name ([crewPresetUnder]). +func crewStoredAt(profileDir, family string) string { + defaults := crewTableFor(family)[DefaultCrew] + stored := make(map[string]string, len(ModelTiers)) + for _, tier := range ModelTiers { + value, _, source, cleared := crewRow(profileDir, tier) + switch { + case cleared: + stored[tier] = "" + case source == "": + stored[tier] = strings.ToLower(strings.TrimSpace(defaults[tier])) + case IsAuto(value): + stored[tier] = AutoValue + default: + stored[tier] = strings.ToLower(strings.TrimSpace(value)) + } + } + table := crewTableFor(family) + // THE DEFAULT PRESET WINS EVERY TIE, the law [crewPresetUnder] states: a + // profile whose every row says auto is a crew with no opinion of its own, + // and it reads balanced rather than whichever preset the loop met first. + if crewStoredMatches(stored, table[DefaultCrew]) { + return DefaultCrew + } + for _, preset := range CrewPresets { + if preset != DefaultCrew && crewStoredMatches(stored, table[preset]) { + return preset + } + } + return CrewCustom +} + +// crewStoredMatches compares a stored reading with one preset row, skipping +// the tiers whose row says auto. +func crewStoredMatches(stored, preset map[string]string) bool { + for _, tier := range ModelTiers { + if stored[tier] == AutoValue { + continue + } + if stored[tier] != strings.ToLower(strings.TrimSpace(preset[tier])) { + return false + } + } + return true +} + // sameCrew compares two crews class by class, case-folded, because a model id is // matched case-insensitively everywhere else on this surface. func sameCrew(live, preset map[string]string) bool { @@ -386,7 +587,7 @@ func SetCrewSource(profileDir, source string) error { // CrewSummary is the one line a crew change confirms itself with: // -// crew → balanced · brain kimi-k3 · hands glm-5.3-flash · checks kimi-k3 +// crew → balanced · brain claude-opus-5 · hands glm-5.3-flash · checks claude-fable-5.1 // // The three names are the classes a person actually asked about — what thinks, // what works, what checks — and HANDS IS THE WORKER: the seat that does the @@ -396,12 +597,38 @@ func SetCrewSource(profileDir, source string) error { // them would be facts that never vary. The ids are shortened to their base names // because the vendor prefix is the half nobody reads twice. func CrewSummary(profileDir string) string { - return "crew → " + CrewAt(profileDir) + " · " + CrewClasses(profileDir) + return crewSummaryWith(profileDir, "") +} + +// CrewSummaryPick is the confirmation with the pick named when it is not the +// default one: +// +// crew → balanced · learn · brain claude-opus-5 · hands glm-5.3-flash · checks claude-fable-5.1 +// +// The pick rides the preset word because the two are one decision read at two +// heights — how much to spend, and where the models for that money come from +// — and a confirmation that said only `balanced` would drop the half the +// person just changed. At the default pick this is [CrewSummary] itself, so a +// profile nobody has taught the pick to confirms exactly as it always has. +func CrewSummaryPick(profileDir string) string { + return crewSummaryWith(profileDir, CrewPickAt(profileDir)) +} + +// crewSummaryWith is the line both summaries are built from: the pick named +// between the preset and the three classes when one was given that is not the +// default, and never otherwise — a profile at the default pick confirms in +// the words it has always confirmed in. +func crewSummaryWith(profileDir, pick string) string { + head := "crew → " + CrewAt(profileDir) + if pick != "" && pick != CrewPickTable { + head += " · " + pick + } + return head + " · " + CrewClasses(profileDir) } // CrewClasses is the three class names alone: // -// brain kimi-k3 · hands glm-5.3-flash · checks kimi-k3 +// brain claude-opus-5 · hands glm-5.3-flash · checks claude-fable-5.1 // // It is the tail of [CrewSummary] lifted out because a second surface prints the // crew now — /status, where the word already has a label of its own and "crew →" @@ -417,7 +644,7 @@ func CrewClasses(profileDir string) string { // CrewClassModels is the three ids [CrewClasses] names, in that order and // without the role words in front of them: // -// kimi-k3, glm-5.3-flash, kimi-k3 +// claude-opus-5, glm-5.3-flash, claude-fable-5.1 // // It exists because a surface drawing the crew line has to be able to say which // runs of it are the ANSWER — the ids a person typed /crew to change — and which diff --git a/internal/config/crew_test.go b/internal/config/crew_test.go index 017a406845..70b2c20a4f 100644 --- a/internal/config/crew_test.go +++ b/internal/config/crew_test.go @@ -38,32 +38,36 @@ func TestTheShippedTierDefaultsAreExactlyTheBalancedCrew(t *testing.T) { } } +// THE OPEN FAMILY IS LOCKED TOO, and this is its page. It is read through +// [CrewModelsForSource] and not [CrewModels], because the bare spelling answers +// the DEFAULT family and the default family is the catalog-wide one — the all +// table's own page is [TestTheAllFamilyNamesTheLockedModels]. func TestCrewPresetsNameTheApprovedModels(t *testing.T) { want := map[string]map[string]string{ CrewFrugal: { ModelTierReflex: "mistralai/mistral-nemo", ModelTierLow: "deepseek/deepseek-v4-flash-0731", - ModelTierWorker: "deepseek/deepseek-v4-flash-0731", + ModelTierWorker: "z-ai/glm-5.3-flash", ModelTierHigh: "z-ai/glm-5.3-flash", - ModelTierMastermind: "z-ai/glm-5.3", + ModelTierMastermind: "z-ai/glm-5.3-flash", }, CrewBalanced: { ModelTierReflex: "mistralai/mistral-nemo", ModelTierLow: "deepseek/deepseek-v4-flash-0731", ModelTierWorker: "z-ai/glm-5.3-flash", ModelTierHigh: "moonshotai/kimi-k3", - ModelTierMastermind: "moonshotai/kimi-k3", + ModelTierMastermind: "z-ai/glm-5.3", }, CrewMax: { ModelTierReflex: "mistralai/mistral-nemo", ModelTierLow: "deepseek/deepseek-v4-flash-0731", ModelTierWorker: "z-ai/glm-5.3", ModelTierHigh: "moonshotai/kimi-k3", - ModelTierMastermind: "moonshotai/kimi-k3", + ModelTierMastermind: "z-ai/glm-5.3", }, } for preset, expected := range want { - got, ok := CrewModels(preset) + got, ok := CrewModelsForSource(CrewSourceOpen, preset) if !ok { t.Fatalf("there is no %s preset", preset) } @@ -213,11 +217,12 @@ func TestTheCrewRowRefusesAWordThatIsNotAPreset(t *testing.T) { // ── the family the three words draw from ──────────────────────────────────── -// THE CAREFUL SEAT IS ALWAYS A SECOND VENDOR, IN EVERY PRESET OF EVERY FAMILY. +// THE CAREFUL SEAT IS A SECOND VENDOR, IN EVERY PRESET OF THE ALL FAMILY. // A check from the same vendor as the work it checks is a check that shares -// that vendor's blind spots, and the two tables are one law: a preset that -// moved to the all family and put the same vendor on worker and careful would -// break the property the open crew was built on while pretending to keep it. +// that vendor's blind spots. The open family's frugal row is the one standing +// exception: worker and careful both sit on glm-5.3-flash there, because at +// that preset's bill the open-weight front has no second vendor to take the +// careful seat. func TestTheCarefulSeatIsAlwaysASecondVendorInEveryFamily(t *testing.T) { vendor := func(id string) string { if at := strings.Index(id, "/"); at > 0 { @@ -227,6 +232,9 @@ func TestTheCarefulSeatIsAlwaysASecondVendorInEveryFamily(t *testing.T) { } for _, family := range CrewSources { for _, preset := range CrewPresets { + if family == CrewSourceOpen && preset == CrewFrugal { + continue + } models, ok := CrewModelsForSource(family, preset) if !ok { t.Fatalf("there is no %s preset in the %s family", preset, family) @@ -248,23 +256,23 @@ func TestTheAllFamilyNamesTheLockedModels(t *testing.T) { CrewFrugal: { ModelTierReflex: "google/gemini-2.5-flash", ModelTierLow: "deepseek/deepseek-v4-flash-0731", - ModelTierWorker: "openai/gpt-5.6-sol", - ModelTierHigh: "google/gemini-3.8-flash", - ModelTierMastermind: "anthropic/claude-opus-5", + ModelTierWorker: "z-ai/glm-5.3-flash", + ModelTierHigh: "qwen/qwen3.8-max-0902", + ModelTierMastermind: "z-ai/glm-5.3-flash", }, CrewBalanced: { ModelTierReflex: "google/gemini-2.5-flash", ModelTierLow: "deepseek/deepseek-v4-flash-0731", - ModelTierWorker: "openai/gpt-5.6-sol", - ModelTierHigh: "anthropic/claude-opus-5", - ModelTierMastermind: "anthropic/claude-fable-5.1", + ModelTierWorker: "z-ai/glm-5.3-flash", + ModelTierHigh: "anthropic/claude-fable-5.1", + ModelTierMastermind: "anthropic/claude-opus-5", }, CrewMax: { ModelTierReflex: "google/gemini-2.5-flash", ModelTierLow: "deepseek/deepseek-v4-flash-0731", - ModelTierWorker: "anthropic/claude-fable-5.1", - ModelTierHigh: "openai/gpt-6-astra", - ModelTierMastermind: "anthropic/claude-fable-5.1", + ModelTierWorker: "z-ai/glm-5.3", + ModelTierHigh: "anthropic/claude-fable-5.1", + ModelTierMastermind: "anthropic/claude-opus-5", }, } for preset, expected := range want { @@ -289,14 +297,14 @@ func TestTheAllFamilyNamesTheLockedModels(t *testing.T) { // pick is the contradiction the chooser exists to prevent. wantLines := map[string]map[string]string{ CrewSourceOpen: { - CrewFrugal: "deepseek works, glm-5.3 thinks · pennies a day", - CrewBalanced: "glm-flash works, kimi-k3 checks and thinks", - CrewMax: "glm-5.3 works, kimi-k3 thinks and checks", + CrewFrugal: "glm-flash works, checks and thinks · pennies a day", + CrewBalanced: "glm-flash works, kimi-k3 checks, glm-5.3 thinks", + CrewMax: "glm-5.3 works and thinks, kimi-k3 checks", }, CrewSourceAll: { - CrewFrugal: "gpt-5.6-sol works, gemini-flash checks, opus thinks", - CrewBalanced: "gpt-5.6-sol works, opus checks, fable thinks", - CrewMax: "fable works and thinks, astra checks", + CrewFrugal: "glm-flash works and thinks, qwen-max checks", + CrewBalanced: "glm-flash works, fable checks, opus thinks", + CrewMax: "glm-5.3 works, fable checks, opus thinks", }, } for family, lines := range wantLines { @@ -308,12 +316,13 @@ func TestTheAllFamilyNamesTheLockedModels(t *testing.T) { } } -// A SOURCE NOBODY SET IS THE OPEN FAMILY. The all table may exist, but a profile -// that never answered the row resolves the open family: the shipped defaults, -// the open presets under the same words, and a write that lands the open ids. -// This is the default-preservation law for the FAMILY. An unknown word is held -// to the same law, silently, the way a retired choice reads everywhere else. -func TestASourceNobodyAnsweredReadsTheOpenFamily(t *testing.T) { +// A SOURCE NOBODY SET IS THE DEFAULT FAMILY, which is the catalog-wide one. The +// open table exists beside it, but a profile that never answered the row +// resolves `all`: the shipped defaults, the all presets under the same words, +// and a write that lands the all ids. This is the default-preservation law for +// the FAMILY. An unknown word is held to the same law, silently, the way a +// retired choice reads everywhere else. +func TestASourceNobodyAnsweredReadsTheDefaultFamily(t *testing.T) { dir := t.TempDir() if got := CrewSourceAt(dir); got != DefaultCrewSource { t.Fatalf("an untouched profile reads the source as %q, want %q", got, DefaultCrewSource) @@ -324,20 +333,21 @@ func TestASourceNobodyAnsweredReadsTheOpenFamily(t *testing.T) { if err := ApplyCrew(dir, CrewMax); err != nil { t.Fatal(err) } - want, _ := CrewModels(CrewMax) + want, _ := CrewModelsForSource(DefaultCrewSource, CrewMax) for _, tier := range ModelTiers { if got := TierModelAt(dir, tier); got != want[tier] { - t.Errorf("with no source set, max wrote %s to %q, want the open %q", tier, got, want[tier]) + t.Errorf("with no source set, max wrote %s to %q, want the default family's %q", tier, got, want[tier]) } } - // A WORD THIS BUILD DOES NOT KNOW READS AS OPEN, silently. Being told a - // preference from an older build is now an error is the one thing a - // retirement must never do, and a family is not a thing to guess either. + // A WORD THIS BUILD DOES NOT KNOW READS AS THE DEFAULT FAMILY, silently. + // Being told a preference from an older build is now an error is the one + // thing a retirement must never do, and a family is not a thing to guess + // either. if err := writeProfileValue(dir, KeyCrewSource, "wide"); err != nil { t.Fatal(err) } - if got := CrewSourceAt(dir); got != CrewSourceOpen { - t.Fatalf("an unknown source word reads as %q, want %q", got, CrewSourceOpen) + if got := CrewSourceAt(dir); got != DefaultCrewSource { + t.Fatalf("an unknown source word reads as %q, want %q", got, DefaultCrewSource) } if got := CrewAt(dir); got != CrewMax { t.Fatalf("an unknown source word moved the crew to %q", got) @@ -388,7 +398,7 @@ func TestFlippingToAllMakesTheWordsMeanTheAllFamily(t *testing.T) { if got := CrewAt(dir); got != CrewBalanced { t.Fatalf("the crew reads %q after balanced was applied under open again", got) } - open, _ := CrewModels(CrewBalanced) + open, _ := CrewModelsForSource(CrewSourceOpen, CrewBalanced) for _, tier := range ModelTiers { if got := TierModelAt(dir, tier); got != open[tier] { t.Errorf("after re-applying under open, %s reads %q, want the open %q", tier, got, open[tier]) @@ -526,3 +536,108 @@ func TestAnUnwrittenSeatFollowsTheFamily(t *testing.T) { t.Fatalf("an all profile with no rows reads the crew as %q, want %q", got, CrewBalanced) } } + +// ── the pick row ──────────────────────────────────────────────────────────── + +// THE PICK ROW DEFAULTS TO THE TABLE, and a word that is not one of the three +// is refused the way every choice row refuses one. +func TestCrewPickDefaultsToTheTable(t *testing.T) { + dir := t.TempDir() + if got := CrewPickAt(dir); got != CrewPickTable { + t.Fatalf("an untouched profile reads the pick %q, want %q", got, CrewPickTable) + } + if got := CrewAt(dir); got != DefaultCrew { + t.Fatalf("an untouched profile reads the crew %q, want %q", got, DefaultCrew) + } + if err := SetCrewPick(dir, CrewPickLearn); err != nil { + t.Fatalf("learn would not set: %v", err) + } + if got := CrewPickAt(dir); got != CrewPickLearn { + t.Fatalf("the pick reads %q after learn was set", got) + } + if err := SetCrewPick(dir, "measured"); err == nil { + t.Fatal("a word that is not one of the three was accepted") + } + if got := CrewPickAt(dir); got != CrewPickLearn { + t.Fatalf("a refused write left the pick reading %q", got) + } +} + +// THE AUTO ROW IS THE OTHER WAY A SEAT IS COMPUTED FROM THE CATALOG'S ROWS, +// beside the pick row this section's tests read. A pick off the table and a +// tier row that says auto both compute from the rows the process holds, and +// the door that waits for them asks [AnyTierAutoAt] for the second. A row +// cleared on purpose says nothing about models, and a crew of model ids asks +// for nothing — but the word reaches the read through the lineage too, so a +// profile older than the worker seat counts its small-work row as well. +func TestAnyTierAutoAtAnswersWhenARowSaysAuto(t *testing.T) { + dir := t.TempDir() + if AnyTierAutoAt(dir) { + t.Fatal("an untouched profile has no auto row to wait for") + } + if err := ApplyCrew(dir, CrewBalanced); err != nil { + t.Fatal(err) + } + if AnyTierAutoAt(dir) { + t.Fatal("a crew of model ids has no auto row to wait for") + } + if err := writeProfileValue(dir, tierKeyFor(ModelTierWorker), AutoValue); err != nil { + t.Fatal(err) + } + if !AnyTierAutoAt(dir) { + t.Fatal("a worker row that says auto went unnoticed") + } + if err := writeProfileValue(dir, tierKeyFor(ModelTierWorker), ""); err != nil { + t.Fatal(err) + } + if AnyTierAutoAt(dir) { + t.Fatal("a row cleared on purpose read as an auto row") + } + vintage := t.TempDir() + if err := writeProfileValue(vintage, tierKeyFor(ModelTierLow), AutoValue); err != nil { + t.Fatal(err) + } + if !AnyTierAutoAt(vintage) { + t.Fatal("an auto row reached through the lineage went unnoticed") + } +} + +// THE PICK-AWARE CREW WORD READS THE BUDGET THE SEATS ARE COMPUTED AT. The +// live seats are computed ids the preset tables do not hold, so the word is +// read off the STORED rows — a profile that applied a crew and then set the +// pick keeps saying the crew it picked — and a row the person emptied on +// purpose still reads custom, because "follows the conversation" is not any +// of the three. +func TestTheCrewWordReadsThePresetThePickComputesAt(t *testing.T) { + dir := t.TempDir() + if err := ApplyCrew(dir, CrewMax); err != nil { + t.Fatal(err) + } + if err := SetCrewPick(dir, CrewPickLearn); err != nil { + t.Fatal(err) + } + if got := CrewAt(dir); got != CrewMax { + t.Fatalf("a max crew under learn reads %q, want %q", got, CrewMax) + } + // And a hand-typed row makes the word custom, as it always has: the stored + // comparison is the honest one, not a second way to say balanced. + rows, err := os.ReadFile(BudgetConfigPath(dir)) + if err != nil { + t.Fatal(err) + } + values := map[string]string{} + if err := json.Unmarshal(rows, &values); err != nil { + t.Fatal(err) + } + values[tierKeyFor(ModelTierMastermind)] = "openai/gpt-5" + raw, err := json.Marshal(values) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(BudgetConfigPath(dir), raw, 0o600); err != nil { + t.Fatal(err) + } + if got := CrewAt(dir); got != CrewCustom { + t.Fatalf("a hand-typed row under learn reads the crew %q, want custom", got) + } +} diff --git a/internal/config/derivation_test.go b/internal/config/derivation_test.go index 55bdfac0e0..0ba5c24fc2 100644 --- a/internal/config/derivation_test.go +++ b/internal/config/derivation_test.go @@ -126,6 +126,15 @@ var settingReaders = map[string]string{ KeyDocumentEngine: "DocumentEngine", KeyVisionModel: "VisionModel", KeyAttribution: "Attribution", + KeyModelPool: "ModelPoolSettingAt", + // The pool key row names the resolver that reads it: [ModelPoolAt] carries + // the word into poolcfg beside the mode, and the puller's keys resolve + // from there (cmd/codeaf's poolTrustedKeys). + KeyModelPoolPublicKey: "ModelPoolAt", + // The crew's pick row names its resolver: the surfaces read the pick to + // say it beside the crew word, and the ladder turns it into computed seats + // ([pickedSeat]) — the identifier proves the word reaches a surface. + KeyCrewPick: "CrewPickAt", // The prompt-profile row names the session field the word lands in, which // is what the far side actually reads: cmd/codeaf's applyV3Governance puts // PromptProfileAt's answer there and internal/session's promptprofile.go diff --git a/internal/config/seats.go b/internal/config/seats.go index 71b0e5a90a..84d29275ab 100644 --- a/internal/config/seats.go +++ b/internal/config/seats.go @@ -65,6 +65,26 @@ const ( // difference between "the row you wrote" and "the row your row was split out // of". SeatInherited SeatSource = "inherited" + // SeatComputed is a tier row that says `auto` answering from the catalog: + // the seat's model computed off the published figures at read time + // ([AutoPick]). It is a rung of the crew's own, not the build's default — + // the person wrote the row — and it says so because a receipt that called it + // `crew` would hide the one fact a reader of the run is checking: the id was + // derived, not named. + SeatComputed SeatSource = "computed" + // SeatTable is a tier row that says `auto` answering from the family's + // table row ([autoRow], [pickedModel]) because nothing could be computed — + // no catalog, or no pick off it. The id is the preset's own, and the rung + // says which kind of answer it was. + SeatTable SeatSource = "table" + // SeatLearned is a seat computed at the crew's budget under the `learn` + // pick ([CrewPickAt]): the catalog's own figures, plus the Model Pool's + // measurements and the person's own judged runs carried as a quality + // prior. It is a rung of its own, beside [SeatComputed], because the one + // fact a reader of a run is checking — was this id derived, and from + // what — is a different answer under the two words: the catalog alone, + // or the catalog plus what runs measured. + SeatLearned SeatSource = "learned" // SeatDefault is this build's choice, for a profile that has never said // anything about models at all. SeatDefault SeatSource = "default" @@ -244,6 +264,26 @@ func tierSeatUnder(profileDir, family, tier string) Seat { model = defaultTierModel(family, tier) } } + // A row that says auto resolves through the one seam both ladders share + // ([autoRow]). Only a row somebody WROTE — directly, or through the lineage + // — can say it: the default rung names a model id, and a cleared row names + // nothing at all. + if IsAuto(model) { + var preset string + model, source, preset = autoRow(profileDir, family, tier) + return Seat{Role: tierSeatRole(tier), Model: model, Source: source, Crew: preset} + } + // AND THE PICK ROW ANSWERS FOR THE SEATS NOBODY NAMED, before the row's + // own rung is read: a pick of catalog or learn computes the dial seats at + // the crew's preset ([pickedSeat]), and leaves everything it does not + // answer — a hand-typed id, a cleared row, the two seats that always read + // the table — exactly as it was. A cleared row is a deliberate answer + // ("follow the conversation") and no pick unsays it on this surface. + if !cleared { + if seat, ok := pickedSeat(profileDir, family, tier, model); ok { + return seat + } + } return Seat{Role: tierSeatRole(tier), Model: model, Source: source, From: from} } @@ -272,8 +312,12 @@ type Seat struct { From string // Crew is the preset word the profile's five tier rows make — `frugal`, // `balanced`, `max` or `custom` ([CrewAt]) — and is empty unless Source is - // [SeatCrew]. It is read rather than stored, exactly as the settings sheet - // reads it, so a receipt and the sheet cannot disagree about which crew ran. + // [SeatCrew], [SeatInherited], [SeatComputed] or [SeatTable]. It is read + // rather than stored, exactly as the settings sheet reads it, so a receipt + // and the sheet cannot disagree about which crew ran. On the computed and + // table rungs it is the preset the auto row is computed at, read from the + // stored rows ([crewPresetUnder]) rather than derived through the resolver, + // which would be the seam asking itself. Crew string } @@ -313,6 +357,12 @@ func (s Seat) Rung() string { // because a receipt that said only `crew custom` would hide exactly the // substitution this rung exists to report. return s.crewRung() + ", inherited" + case SeatComputed: + return s.crewRung() + ", computed from the catalog" + case SeatLearned: + return s.crewRung() + ", learned" + case SeatTable: + return s.crewRung() + ", table" } return "default" } @@ -411,7 +461,7 @@ func (s Seat) Report() string { return withNotice(s.Line(), s.Notice()) } // Describe is one seat in a receipt's voice: // -// work deepseek/deepseek-v4-flash-0731 (crew frugal) +// work z-ai/glm-5.3-flash (crew frugal) // plan follows the work model (default) // // THE EMPTINESS LAW, as the crew row already keeps it: an unfilled plan seat is @@ -441,7 +491,7 @@ type Seats struct { // Sentence is both seats, unlabelled, for a door whose opening lines have a // label column of their own: // -// work deepseek/deepseek-v4-flash-0731 (crew frugal) · plan z-ai/glm-5.3 (crew frugal) +// work z-ai/glm-5.3-flash (crew frugal) · plan z-ai/glm-5.3-flash (crew frugal) // // ONE SHAPE AND NOT TWO. It would read a little better to collapse a run whose // seats came from the same crew into one clause, and it would mean a script @@ -453,7 +503,7 @@ func (s Seats) Sentence() string { // Line is the one line a headless run opens with: // -// models: work deepseek/deepseek-v4-flash-0731 (crew frugal) · plan z-ai/glm-5.3 (crew frugal) +// models: work z-ai/glm-5.3-flash (crew frugal) · plan z-ai/glm-5.3-flash (crew frugal) func (s Seats) Line() string { return modelsLabel + s.Sentence() } // Notice is the inheritance line for whichever seat was filled by an older row, @@ -570,11 +620,37 @@ func resolveSeat(role SeatRole, profileDir, flag, tier, fallback string) Seat { // fallback except silence: a row cleared on purpose says "follow the // conversation", which a headless run has no conversation to answer with, // and it lands on the same bottom rung a profile that said nothing does. - model, from, source, _ := crewRow(profileDir, tier) + model, from, source, cleared := crewRow(profileDir, tier) if source == "" { + // THE PICK ANSWERS BEFORE THE BOTTOM RUNG: an unwritten seat under a + // pick of catalog or learn is computed at the crew's preset + // ([pickedSeat]), the same answer the conversation reads, and a row + // cleared on purpose still falls through — the pick answers for seats + // nobody named, and a cleared row is somebody naming emptiness. + if !cleared { + if seat, ok := pickedSeat(profileDir, CrewSourceAt(profileDir), tier, ""); ok { + return seat + } + } seat.Model, seat.Source = fallback, SeatDefault return seat } + // A row that says auto resolves through the one seam both ladders share + // ([autoRow]) — on the rung it earned, with the preset it was computed at + // carried for the receipt — and never to `auto` and never to empty, which + // is what a headless run could do nothing with. + if IsAuto(model) { + var preset string + seat.Model, seat.Source, preset = autoRow(profileDir, CrewSourceAt(profileDir), tier) + seat.Crew = preset + return seat + } + // AND BETWEEN THE ROW AND ITS OWN RUNG, the pick: a written row holding + // the preset's own table value is the preset answering rather than a pin, + // and the pick computes it ([pickedSeat]); a hand-typed id keeps its rung. + if seat, ok := pickedSeat(profileDir, CrewSourceAt(profileDir), tier, model); ok { + return seat + } seat.Model, seat.Source, seat.From, seat.Crew = model, source, from, CrewAt(profileDir) return seat } diff --git a/internal/config/seats_test.go b/internal/config/seats_test.go index d863b3e2ba..977aa960c2 100644 --- a/internal/config/seats_test.go +++ b/internal/config/seats_test.go @@ -14,6 +14,8 @@ import ( "testing" "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/crewpick" "github.com/Agent-Field/codeaf/internal/router" ) @@ -43,8 +45,8 @@ func TestTheSeatLadderAnswersInItsOwnOrder(t *testing.T) { { name: "the crew answers when it is the only thing said", crew: CrewFrugal, - work: "deepseek/deepseek-v4-flash-0731", workRung: "crew frugal", - plan: "z-ai/glm-5.3", planRung: "crew frugal", + work: "z-ai/glm-5.3-flash", workRung: "crew frugal", + plan: "z-ai/glm-5.3-flash", planRung: "crew frugal", }, { name: "the environment outranks the crew", @@ -79,7 +81,7 @@ func TestTheSeatLadderAnswersInItsOwnOrder(t *testing.T) { name: "the crew leaves generation behavior to the model", crew: CrewMax, work: "z-ai/glm-5.3", workRung: "crew max", - plan: "moonshotai/kimi-k3", planRung: "crew max", + plan: "anthropic/claude-opus-5", planRung: "crew max", }, { // And a flag carrying one is not shortened either, so the two rungs @@ -98,7 +100,7 @@ func TestTheSeatLadderAnswersInItsOwnOrder(t *testing.T) { // it: the same row a task handed off in conversation rides. hand: map[string]string{ModelTierWorker: "vendor/my-own-worker"}, work: "vendor/my-own-worker", workRung: "crew custom", - plan: "z-ai/glm-5.3", planRung: "crew custom", + plan: "z-ai/glm-5.3-flash", planRung: "crew custom", }, { // A row cleared on purpose means "follow the conversation", and a @@ -841,3 +843,121 @@ func TestTheReceiptNamesTheSpellingOfTheVariableThatAnswered(t *testing.T) { t.Fatalf("rung = %q model = %q, want %q and its value", got, seats.Work.Model, ModelEnv) } } + +// ── the pick row's half of the ladder ─────────────────────────────────────── + +// A PICK OFF THE TABLE COMPUTES THE DIAL SEATS AND LEAVES THE OTHER TWO ALONE. +// The three seats that the presets dial — worker, careful work, mastermind — +// are computed at the crew's preset, `learned` under the learn word and +// `computed from the catalog` under the catalog word; reflex and small work +// always read the table, which is what makes a pick a dial over three seats +// and not a second crew. The ids are asserted against crewpick's own front, +// the way the shipped tables are (auto_test.go owns the method). +func TestAPickOffTheTableComputesTheDialSeats(t *testing.T) { + restore := AutoModels + AutoModels = func() []catalog.Model { return autoTestRows() } + defer func() { AutoModels = restore }() + + _, balanced, _ := crewpick.Presets(crewpick.Front(autoCandidates(autoTestRows()), crewpick.DefaultShapes(), crewpick.All)) + table, _ := CrewModelsForSource(CrewSourceAt(""), DefaultCrew) + + for _, pick := range []struct { + word string + source SeatSource + }{ + {CrewPickLearn, SeatLearned}, + {CrewPickCatalog, SeatComputed}, + } { + dir := writeProfileRows(t, map[string]string{KeyCrewPick: pick.word}) + for _, c := range []struct { + tier string + want string + }{ + {ModelTierWorker, balanced.Worker}, + {ModelTierHigh, balanced.High}, + {ModelTierMastermind, balanced.Mastermind}, + } { + seat := TierSeatAt(dir, c.tier) + if seat.Model != c.want || seat.Source != pick.source { + t.Errorf("%s: the %s seat reads %q (%s), want %q (%s)", pick.word, c.tier, + seat.Model, seat.Source, c.want, pick.source) + } + if seat.Crew != DefaultCrew { + t.Errorf("%s: the %s seat ran at %q, want the untouched profile's %q", pick.word, c.tier, seat.Crew, DefaultCrew) + } + } + // THE PICK COMPUTES RATHER THAN READS: the front's ids are not the + // preset table's own, so a seat carrying the front's id is a seat the + // catalog answered and not the table in disguise. + if balanced.Worker == table[ModelTierWorker] { + t.Fatal("the test rows field a front identical to the shipped table; the pick would be asserting nothing") + } + // REFLEX AND SMALL WORK ALWAYS READ THE TABLE, and a pick does not + // touch them: the same rung an untouched profile has always read, with + // the table's own id. + for _, tier := range []string{ModelTierReflex, ModelTierLow} { + seat := TierSeatAt(dir, tier) + if seat.Model != table[tier] { + t.Errorf("%s: the %s seat reads %q, want the table's %q", pick.word, tier, seat.Model, table[tier]) + } + if seat.Source != SeatDefault { + t.Errorf("%s: the %s seat reads rung %q, want the table's own default", pick.word, tier, seat.Source) + } + } + } +} + +// A HAND-TYPED MODEL SURVIVES THE PICK. The pick answers for the seats nobody +// named, not over the names they typed — and the headless ladder says the same +// thing the conversation says, because a pick that meant one thing in chat and +// another under `codeaf do` would be the divergence the ladder exists to +// prevent. +func TestAHandTypedSeatSurvivesThePick(t *testing.T) { + restore := AutoModels + AutoModels = func() []catalog.Model { return autoTestRows() } + defer func() { AutoModels = restore }() + + dir := writeProfileRows(t, map[string]string{ + KeyCrewPick: CrewPickLearn, + tierKeyFor(ModelTierWorker): "someone/else", + }) + + seat := TierSeatAt(dir, ModelTierWorker) + if seat.Model != "someone/else" || seat.Source != SeatCrew { + t.Fatalf("a hand-typed worker reads %q (%s), want someone/else (crew)", seat.Model, seat.Source) + } + // The seats the pick does answer are computed at the preset the stored + // rows still make — the hand-typed row costs them their preset match, so + // the budget falls back to the default preset, which is what an undecided + // profile runs at. + _, balanced, _ := crewpick.Presets(crewpick.Front(autoCandidates(autoTestRows()), crewpick.DefaultShapes(), crewpick.All)) + high := TierSeatAt(dir, ModelTierHigh) + if high.Model != balanced.High || high.Source != SeatLearned { + t.Fatalf("the careful seat reads %q (%s), want %q (learned)", high.Model, high.Source, balanced.High) + } + + // AND THE HEADLESS LADDER SAYS THE SAME. The work seat is the worker tier + // and the plan seat is the mastermind tier, and both must answer exactly + // what the conversation's rows answer — the receipt's own claim. + seats := ResolveSeats(dir, "", "") + if seats.Work.Model != "someone/else" || seats.Work.Source != SeatCrew { + t.Fatalf("headless, the work seat reads %q (%s), want someone/else (crew)", seats.Work.Model, seats.Work.Source) + } + if seats.Plan.Model != balanced.Mastermind || seats.Plan.Source != SeatLearned { + t.Fatalf("headless, the plan seat reads %q (%s), want %q (learned)", seats.Plan.Model, seats.Plan.Source, balanced.Mastermind) + } +} + +// THE PICK'S RUNGS SAY THEIR WORDS, so a receipt never has to be guessed at: +// the catalog word computes, the learn word learned, and the table's own +// fallback keeps its one word. +func TestThePickRungsSayTheirWords(t *testing.T) { + seat := Seat{Crew: CrewBalanced, Source: SeatComputed} + if got := seat.Rung(); got != "crew "+CrewBalanced+", computed from the catalog" { + t.Errorf("the computed rung reads %q, want crew %s, computed from the catalog", got, CrewBalanced) + } + seat.Source = SeatLearned + if got := seat.Rung(); got != "crew "+CrewBalanced+", learned" { + t.Errorf("the learned rung reads %q, want crew %s, learned", got, CrewBalanced) + } +} diff --git a/internal/config/settings.go b/internal/config/settings.go index 84ca7a332f..9a397d5f3f 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -12,6 +12,7 @@ import ( "github.com/Agent-Field/codeaf/internal/ctxbudget" "github.com/Agent-Field/codeaf/internal/env" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" "github.com/Agent-Field/codeaf/internal/provider" "github.com/Agent-Field/codeaf/internal/roles" "github.com/Agent-Field/codeaf/internal/search" @@ -111,8 +112,18 @@ const ( KeyTenureAfter = "tenure_after" KeyDocumentEngine = "document_engine" KeyVisionModel = "vision_model" - KeyAttribution = "attribution" - KeySplitPct = "split_pct" + // KeyModelPool is the stored word the pool's resolver takes: the same three + // answers the CODEAF_MODEL_POOL pin and a CI environment may give it + // (internal/pool/poolcfg). The resolver beside the search rows is the one + // place the process environment is read for the pool. + KeyModelPool = "model_pool" + // KeyModelPoolPublicKey is the trusted key a fetched pool index is checked + // under: base64 text beside the pool row it narrows, and empty for the + // key the binary carries. The environment pin CODEAF_MODEL_POOL_PUBLIC_KEY + // outranks it, through the same resolver. + KeyModelPoolPublicKey = "models.pool.public_key" + KeyAttribution = "attribution" + KeySplitPct = "split_pct" // The two rows the v3 chat surface keeps on disk BESIDE the conversation: // what was typed, and what was half-typed. They are one pair of questions — @@ -234,6 +245,18 @@ const ( // worker row's own reason: a repository that could answer it could send a // visitor's work, and their credit, to a vendor they never chose. KeyCrewSource = "models.crew.source" + // KeyCrewPick is where the crew's seats are picked from when a tier row + // does not hold a model id of its own. The three words are read and + // answered by the crew's own machinery (crew.go's [CrewPickAt] and + // [SetCrewPick]) beside the words [KeyCrew] and [KeyCrewSource] take: the + // crew row says how much to spend, the family row says which shelf those + // budgets name, and this row says where the models for that money come + // from — the rows this build measured, or a computation off the catalog + // made again on every read, with or without what the Model Pool measured. + // It is PROFILE-ONLY with the crew and family rows, for the worker row's + // own reason: a repository that could answer it could send a visitor's + // work, and their credit, to a model nobody on that machine chose. + KeyCrewPick = "models.crew.pick" // KeyMouse is whether the surface reports the mouse at all. ON is the // default ([DefaultMouse]), because hover, click and the wheel are v3's own // language and the thing they cost is bought back by a key: an alt-screen @@ -1001,8 +1024,9 @@ const ( var ModelTiers = []string{ModelTierReflex, ModelTierLow, ModelTierWorker, ModelTierHigh, ModelTierMastermind} // THE SHIPPED CREW. All five tiers arrive pointed at a model, and the five -// together are exactly the `balanced` preset (crew.go) — which is what makes the -// crew row read "balanced" on a profile nobody has touched instead of reading +// together are exactly the `balanced` row of the DEFAULT FAMILY (crew.go's +// [crewAllModels], named by [DefaultCrewSource]) — which is what makes the crew +// row read "balanced" on a profile nobody has touched instead of reading // "custom" about its own defaults. // // Each is a bare OpenRouter id, spelled ONCE here and read by every caller @@ -1014,29 +1038,30 @@ var ModelTiers = []string{ModelTierReflex, ModelTierLow, ModelTierWorker, ModelT // to, which is [roles.Resolve]'s floor. UNSET and CLEARED are different answers // here, and that distinction is the whole mechanism ([TierModelAt] says how). // -// The ids are OPEN-WEIGHT MODELS, chosen off the catalog's own published -// scores (OpenRouter republishes Artificial Analysis's coding and agentic -// indexes on every row) against the blended price (crew.go's crewModels comment -// owns the date). The low row is pinned to a DATED build on purpose: the bare `deepseek/deepseek-v4-flash` id -// resolves to the April build, and the July build at the same price scores -// thirteen coding points higher. +// The ids are read off the catalog's own published rows — its intelligence, +// coding and agentic indexes against its prompt, completion and cache-read +// prices, priced under each seat's own call shape (crew.go's [crewAllModels] +// comment owns the method and the date). The low row is pinned to a DATED build +// on purpose: the bare `deepseek/deepseek-v4-flash` id resolves to the April +// build, and the July build costs the same. const ( - DefaultReflexModel = "mistralai/mistral-nemo" + DefaultReflexModel = "google/gemini-2.5-flash" DefaultLowModel = "deepseek/deepseek-v4-flash-0731" - // The worker is the seat that pays most of a task's bill, so the balanced - // crew puts the best agentic score per dollar on it rather than the best - // score: glm-5.3-flash sits one point under glm-5.3 on the agentic index at - // a twentieth of the price, and it can see images, which the parent can hand - // it without a vision detour. + // The worker is the seat that pays most of a task's bill, so it is the last + // seat a preset spends on: a step here multiplies through every token a task + // runs up, where a step on the two low-volume seats is paid a handful of + // times. glm-5.3-flash is the point on the long-cached-loop front that + // balanced runs at, and it can see images, which the parent can hand it + // without a vision detour. DefaultWorkerModel = "z-ai/glm-5.3-flash" // The careful tier is ALWAYS A DIFFERENT VENDOR FROM THE WORKER, in every // preset, and always a model that sees images: a check from a second vendor // catches what the first vendor's blind spots let through, and the vision // role rides this row. - DefaultHighModel = "moonshotai/kimi-k3" + DefaultHighModel = "anthropic/claude-fable-5.1" // The mastermind names a capable planning model. Its generation behavior is // left to the provider unless an operator adds a level to the model id. - DefaultMastermindModel = "moonshotai/kimi-k3" + DefaultMastermindModel = "anthropic/claude-opus-5" ) // DocumentEngines are the four rungs CODEAF_DOC_ENGINE accepts. @@ -1060,6 +1085,10 @@ const SearchProviderAuto = "auto" // jina-search is named here too so every registered search plug is pinnable. var SearchProviders = []string{SearchProviderAuto, "firecrawl", "duckduckgo", "exa", "jina-search"} +// ModelPoolChoices are the answers the pool row accepts, and the same words +// the pool's resolver reads from its environment pin. +var ModelPoolChoices = []string{"on", "read", "off"} + // OperatorEnvPins is the explicit allowlist of environment variables that are // plumbing rather than settings: endpoints, credentials, profile roots, and // planner internals. They are listed read-only in the sheet's environment @@ -1264,6 +1293,25 @@ var OperatorEnvPins = []string{ // shape, under `make demo-home`'s terms. A row offering to persist a // fixture would put a demo question in front of a person every morning. "CODEAF_QUESTION_DEMO", + // The four pool names that stay plumbing (internal/pool/poolcfg, which + // names them and reads none of them: the environment reaches that package + // as a function the caller hands in). The relay is the base the two URLs + // derive from — index.json and /v1/rows under it — and the TTL is how + // long a read copy stays young: endpoints and a cadence, on the terms the + // other addresses here are on. The public key is a row, not plumbing: it + // narrows what the install trusts, which is a preference. + // + // THE WORD ITSELF IS NOT HERE ANY MORE. CODEAF_MODEL_POOL fronts the + // model_pool row now, which is what the comment below used to promise: + // it was listed here because poolcfg took the setting as an argument and + // nothing in the binary called it, and a pin with no row was the honest + // answer then. With the row here, the word renders through the row — dim, + // "pinned by CODEAF_MODEL_POOL" — the way CODEAF_DOC_ENGINE does. + "CODEAF_MODEL_POOL_RELAY_URL", + "CODEAF_MODEL_POOL_URL", + "CODEAF_MODEL_POOL_SUBMIT_URL", + "CODEAF_MODEL_POOL_MIRROR_URL", + "CODEAF_MODEL_POOL_TTL", } // Defaults the registry owns beyond the ones config.go already declares. @@ -1847,6 +1895,34 @@ func (s *Settings) build() []Setting { read: func() string { return SearchProviderAt(dir) }, write: func(raw string) error { return writeChoice(dir, KeySearchProvider, raw, SearchProviders) }, }, + // The Model Pool row. It is the stored word [ModelPoolAt] resolves, and + // the pin beside it is one of the four names the resolver takes. The + // row sits with the search rows because it is the same shape of + // question — what does codeaf reach OUTSIDE the machine for — and its + // answer is read the same way a choice is read everywhere here. + Setting{ + Key: KeyModelPool, Category: CategoryModels, Kind: SettingChoice, + Label: "model pool", Env: "CODEAF_MODEL_POOL", Choices: ModelPoolChoices, + Hint: "codeaf picks your models from the public Model Pool, and your runs improve it. " + + "On by default: what leaves is computed, text-free numbers under a per-install nonce, " + + "never code, prompts or paths. read: use the pool, send nothing. off: neither " + + "(CODEAF_MODEL_POOL=off for one run).", + read: func() string { return ModelPoolAt(dir).Mode.String() }, + write: func(raw string) error { return writeChoice(dir, KeyModelPool, raw, ModelPoolChoices) }, + }, + // The key a fetched pool index is checked under. The binary carries the + // index signer's key, and this row exists to pin a different one — a + // mirror serving its own signed index, or an install proving a key out + // for itself. It sits beside the pool row because it narrows that row's + // reading, the way the keys under searching narrow theirs. + Setting{ + Key: KeyModelPoolPublicKey, Category: CategoryModels, Kind: SettingText, + Label: "pool key", Env: "CODEAF_MODEL_POOL_PUBLIC_KEY", EmptyLabel: "built in", + Hint: "the base64 Ed25519 public key an index must be signed with. " + + "Empty means the key built into this binary.", + read: func() string { return ModelPoolAt(dir).PublicKey }, + write: func(raw string) error { return writeText(dir, KeyModelPoolPublicKey, raw) }, + }, // THE KEY EVERY MODEL CALL RIDES. The default local door normally creates // one through the browser (tui3's firstrun.go); this row remains the place // to paste a replacement or to use a custom endpoint's credential. It masks @@ -2305,19 +2381,37 @@ func (s *Settings) build() []Setting { read: func() string { return CrewAt(dir) }, write: func(raw string) error { return writeCrew(dir, raw) }, }, + // WHERE THE SEATS ARE PICKED FROM, one row under the crew. The crew row + // says how much to spend and this says where the models for that money + // come from when a tier row does not hold a person's own id: the rows + // this build measured and shipped, or the same three budgets recomputed + // off the catalog on every read, with or without what the Model Pool + // and the person's own judged runs measured. The words are the crew's + // own (crew.go), so the row and the ladder cannot disagree about what a + // pick means. + Setting{ + Key: KeyCrewPick, Category: CategoryModels, Kind: SettingChoice, + Label: "picked from", Choices: CrewPicks, + Hint: "where the crew's models come from. table: the rows we measured. " + + "catalog: recomputed from today's published prices and scores at your " + + "crew's budget. learn: catalog plus the Model Pool's measurements and " + + "your own judged runs.", + read: func() string { return CrewPickAt(dir) }, + write: func(raw string) error { return SetCrewPick(dir, raw) }, + }, // THE FAMILY THE THREE WORDS DRAW FROM, one row under the crew. It sits // beside the crew row because it is the same decision read one level up: // the crew row says which five models, and this says which shelf those - // five come off. Open is the default and the law the shipped crew rests - // on; `all` is the opt-in that spends what the frontier costs. + // five come off. `all` is the default and what the shipped five are the + // balanced row of; `open` narrows the same three words to open weights. Setting{ Key: KeyCrewSource, Category: CategoryModels, Kind: SettingChoice, Label: "model family", Choices: CrewSources, - Hint: "which family the crew words draw from. `open` is the default: every " + - "seat an open-weight model, so a crew nobody chose is never a bet on one " + - "vendor's pricing. `all` reads the same three words, frugal, balanced and " + - "max, off the whole catalog, closed and frontier models included, and " + - "costs what those models cost. Seats nobody pinned move with the family at " + + Hint: "which family the crew words draw from. `all` is the default: frugal, " + + "balanced and max read off the whole catalog, closed and frontier models " + + "included, and cost what those models cost. `open` reads the same three " + + "words off the open-weight rows only, so no seat is a bet on one vendor's " + + "pricing. Seats nobody pinned move with the family at " + "once, because an unwritten seat is the default crew; rows already written " + "keep their ids until you pick the crew again.", read: func() string { return CrewSourceAt(dir) }, @@ -3260,6 +3354,33 @@ func SearchProviderAt(profileDir string) string { return DefaultSearchProvider } +// ModelPoolSettingAt is the stored word the pool row holds: one of the three +// choices, or empty for a person who has never answered. It is the argument +// the pool's resolver takes, and the raw half of [ModelPoolAt]. +func ModelPoolSettingAt(profileDir string) string { + value, _ := persistedString(profileDir, KeyModelPool) + return value +} + +// ModelPoolPublicKeySettingAt is the stored word the pool key row holds: +// base64 text, or empty for the key the binary carries. It is the second +// setting [ModelPoolAt] resolves. +func ModelPoolPublicKeySettingAt(profileDir string) string { + value, _ := persistedString(profileDir, KeyModelPoolPublicKey) + return value +} + +// ModelPoolAt resolves how the Model Pool behaves: the stored mode word, the +// stored public key and the pool's environment names, through poolcfg.Resolve. +// IT IS THE ONE PLACE THE PROCESS ENVIRONMENT IS READ FOR THE POOL — poolcfg +// itself takes the environment as a function, and a caller with its own (the +// `codeaf pool` verb, whose tests inject one) resolves [ModelPoolSettingAt] +// and [ModelPoolPublicKeySettingAt] against its own lookup rather than +// calling this. +func ModelPoolAt(profileDir string) poolcfg.Config { + return poolcfg.Resolve(ModelPoolSettingAt(profileDir), ModelPoolPublicKeySettingAt(profileDir), os.LookupEnv) +} + // ExaKeyAt resolves the Exa credential: the environment first, then the sheet, // then empty — and empty is a working configuration, not a fault. func ExaKeyAt(profileDir string) string { @@ -3826,10 +3947,10 @@ func tierKeyFor(tier string) string { // defaultTierModel is what a tier answers on a profile that has never held its // key: the DEFAULT CREW, resolved in the family the profile chose. It is not the // build's constant alone, because a profile that answered the family row and no -// tier row would otherwise run the open-weight crew under a frontier label: the -// family saying one thing and the ladder another. Under the shipped family the +// tier row would otherwise run one family's crew under the other's label: the +// family saying one thing and the ladder another. Under the default family the // answer is the constant, which is what [DefaultWorkerModel] and its kin name, -// because the open table's default row and the builtin five name the same set by +// because that family's default row and the builtin five name the same set by // construction (crew_test.go pins it). func defaultTierModel(family, tier string) string { if table, ok := CrewModelsForSource(family, DefaultCrew); ok { diff --git a/internal/config/settings_test.go b/internal/config/settings_test.go index 03e68d4f95..24b03f77de 100644 --- a/internal/config/settings_test.go +++ b/internal/config/settings_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/Agent-Field/codeaf/internal/pool/poolcfg" "github.com/Agent-Field/codeaf/internal/standing" "github.com/Agent-Field/codeaf/internal/taxonomy" ) @@ -52,6 +53,52 @@ func TestSlackApplicationRowShowsOnlyThePersonsAnswer(t *testing.T) { } } +// The pool key row: the trusted word a fetched index is checked under. Empty +// is the ordinary answer and reads as the key the binary carries; a stored +// word is the one the install trusts, and the pin outranks it. +func TestThePoolKeyRowReadsBuiltInAndFollowsItsStoredWordAndItsPin(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_MODEL_POOL_PUBLIC_KEY", "") + t.Setenv("CODEAF_MODEL_POOL", "") + rows := registry(t, dir) + row, ok := rows.Row(KeyModelPoolPublicKey) + if !ok { + t.Fatal("the pool key is not registered") + } + if row.Category != CategoryModels || row.Kind != SettingText || row.Label != "pool key" { + t.Fatalf("pool key row = %+v", row) + } + if row.Value() != "built in" { + t.Fatalf("an unset key row reads %q, want built in", row.Value()) + } + if got := ModelPoolAt(dir); got.PublicKey != "" || got.Source.PublicKey != "default" { + t.Fatalf("an untouched profile resolved %+v", got) + } + + if err := row.Apply("A-base64-word="); err != nil { + t.Fatal(err) + } + reread, _ := registry(t, dir).Row(KeyModelPoolPublicKey) + if reread.Value() != "A-base64-word=" { + t.Fatalf("the reread row lost the persisted word: %q", reread.Value()) + } + if got := ModelPoolAt(dir); got.PublicKey != "A-base64-word=" || got.Source.PublicKey != "setting" { + t.Fatalf("the stored key did not take: %+v", got) + } + + t.Setenv("CODEAF_MODEL_POOL_PUBLIC_KEY", "a-pinned-key") + pinned, _ := registry(t, dir).Row(KeyModelPoolPublicKey) + if name, isPinned := pinned.PinnedBy(); !isPinned || name != "CODEAF_MODEL_POOL_PUBLIC_KEY" { + t.Fatalf("the key row did not report its pin: %q %v", name, isPinned) + } + if got := ModelPoolAt(dir); got.PublicKey != "a-pinned-key" || got.Source.PublicKey != "env" { + t.Fatalf("the pin lost: %+v", got) + } + if err := pinned.Apply("another"); err == nil || !strings.Contains(err.Error(), "CODEAF_MODEL_POOL_PUBLIC_KEY") { + t.Fatalf("a pinned key row accepted an edit: %v", err) + } +} + // The registry is the completeness gate: a user-tunable environment pin added // anywhere in the tree has to arrive as a row here, or land on the explicit // operator-plumbing allowlist. Until then this test fails the build. @@ -1381,3 +1428,70 @@ func TestThePromptProfileRowRoundTripsAndItsPinWinsForOneLaunch(t *testing.T) { t.Fatalf("a mistyped pin overrode the persisted row: %q", got) } } + +// The Model Pool row: the stored word the pool resolver takes, the pin beside +// it, and the resolver's own reading of both. The three choices are the same +// three words poolcfg.Resolve accepts from the environment, so a row and a +// shell spell one thing — and the resolver in this package is the one place +// the process environment is read for the pool. +func TestTheModelPoolRowDefaultsToOnAndFollowsItsStoredWordAndItsPin(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_MODEL_POOL", "") + t.Setenv("CI", "") + rows := registry(t, dir) + row, ok := rows.Row(KeyModelPool) + if !ok { + t.Fatal("the model pool is not registered") + } + if row.Category != CategoryModels || row.Kind != SettingChoice || row.Label != "model pool" { + t.Fatalf("model pool row = %+v", row) + } + if row.Value() != "on" { + t.Fatalf("the pool does not default to on: %q", row.Value()) + } + if got := ModelPoolAt(dir); got.Mode != poolcfg.On || got.Source.Mode != "default" { + t.Fatalf("an untouched profile resolved %+v", got) + } + + if err := row.Apply("read"); err != nil { + t.Fatal(err) + } + reread, _ := registry(t, dir).Row(KeyModelPool) + if reread.Value() != "read" { + t.Fatalf("the reread row lost the persisted choice: %q", reread.Value()) + } + if got := ModelPoolAt(dir); got.Mode != poolcfg.Read || got.Source.Mode != "setting" { + t.Fatalf("the stored word did not take: %+v", got) + } + if err := reread.Apply("maybe"); err == nil || !strings.Contains(err.Error(), "on, read, off") { + t.Fatalf("the row accepted a word it does not take: %v", err) + } + if ModelPoolAt(dir).Mode != poolcfg.Read { + t.Fatal("a refused edit still moved the row") + } + + // The pin outranks the stored word, the row reports it, and a pinned row + // refuses to be edited the way every other pinned row refuses. + t.Setenv("CODEAF_MODEL_POOL", "off") + pinned, _ := registry(t, dir).Row(KeyModelPool) + if name, isPinned := pinned.PinnedBy(); !isPinned || name != "CODEAF_MODEL_POOL" { + t.Fatalf("the pool row did not report its pin: %q %v", name, isPinned) + } + if got := ModelPoolAt(dir); got.Mode != poolcfg.Off || got.Source.Mode != "env" { + t.Fatalf("the pin lost: %+v", got) + } + if err := pinned.Apply("on"); err == nil || !strings.Contains(err.Error(), "CODEAF_MODEL_POOL") { + t.Fatalf("a pinned pool row accepted an edit: %v", err) + } + + // A word nobody set the row to on purpose — an older build's word, a + // hand-typed typo — reads as the default rather than as an error, the way + // [SearchProviderAt] reads "yahoo!" as auto. + t.Setenv("CODEAF_MODEL_POOL", "") + if err := writeProfileValue(dir, KeyModelPool, "maybe"); err != nil { + t.Fatal(err) + } + if got := ModelPoolAt(dir); got.Mode != poolcfg.On { + t.Fatalf("a stale word did not fall back to on: %+v", got) + } +} diff --git a/internal/crewpick/crewpick.go b/internal/crewpick/crewpick.go new file mode 100644 index 0000000000..c2eb0cc138 --- /dev/null +++ b/internal/crewpick/crewpick.go @@ -0,0 +1,707 @@ +// Package crewpick picks one model per seat for a three-seat crew by reading +// the pareto front of expected task bill against crew quality. +// +// It deliberately imports nothing from the rest of the tree and touches no +// disk and no network: the caller owns the candidate list and the seat +// shapes, and the same candidates give the same front however often they are +// asked about and in whatever order they arrive. +// +// The law it implements: every (worker, high, mastermind) combination the +// candidates can field is scored by the bill its seats' token volumes run up +// and by the mean of its three seat qualities. A candidate is in the running +// for a seat only when it meets the family and the seat's needs and its +// quality reaches Floor of that seat's best in the family, and a crew is +// refused outright when its high seat comes from the worker's own vendor. +// The crews that no other crew beats on both bill and quality survive as the +// front, sorted by bill; Presets and AtKnob read picks off that front. +// +// A candidate that publishes some but not all of its three indexes is scored +// on the ones it publishes: each missing index is estimated from the call's +// candidates that publish both it and one of its measured ones — the donors — +// never above the call's largest measured value of that index, and the +// result says which of a pick's indexes were estimated. A candidate that +// publishes none of its indexes, or no price at all, is out of the running. +package crewpick + +import ( + "math" + "slices" + "sort" + "strings" +) + +// An Index is one of the three capability indexes quality is read from — +// intelligence, coding, agentic — in the order the seat weights and a +// candidate's reading carry them. +type Index int + +const ( + Intelligence Index = iota + Coding + Agentic +) + +// A Candidate is one model up for a seat. Prices are dollars per 1M tokens. +// +// A candidate carrying none of the three indexes — zero values, which is how +// unpublished indexes read — or carrying a zero prompt price is not a +// candidate at all: it never occupies a seat, never clears a floor, and +// raises no ceiling. A candidate carrying some but not all of its indexes is +// scored on the ones it publishes, each missing one estimated from the +// donors among its fellow candidates, and the result says which of a pick's +// indexes those were. A model that publishes no cache-read price says so with +// HasCacheRead false and pays the prompt price on the cache share too. +type Candidate struct { + // ID names the candidate; the text before its first slash is the vendor. + ID string + // Open marks a candidate the open family may draw from. + Open bool + // Intelligence, Coding and Agentic are the three capability indexes + // quality is read from. + Intelligence float64 + Coding float64 + Agentic float64 + // PromptPrice, CompletionPrice and CacheReadPrice are the dollars per + // 1M tokens of each kind. + PromptPrice float64 + CompletionPrice float64 + CacheReadPrice float64 + // HasCacheRead reports whether CacheReadPrice is published at all. + HasCacheRead bool + // Context is the model's context window, in tokens. + Context int + // Images and Tools are what the model can take and call. + Images bool + Tools bool +} + +// A Seat is one of a crew's three roles: the worker carries the volume of +// the work, the high seat judges and repairs what the worker writes, and the +// mastermind spends a little context steering the whole task. +type Seat int + +const ( + Worker Seat = iota + High + Mastermind +) + +// A SeatShape is how one seat uses whatever model sits it: the weights +// quality is read from, the token mix cost is computed on, what the model must +// be able to take and call, the window it must have, and the share of a +// task's tokens the seat burns. +type SeatShape struct { + // Weights reads quality from the three indexes: intelligence, coding, + // agentic. + Weights [3]float64 + // InOut is the seat's input tokens per output token; the completion + // price is spread over it. + InOut float64 + // CacheShare is the share of the seat's input tokens served from cache. + CacheShare float64 + // NeedTools and NeedImages are what a model must take and call to sit + // the seat. + NeedTools bool + NeedImages bool + // MinContext is the window a model must have to sit the seat. + MinContext int + // Volume is the seat's share of a task's tokens, and the weight its + // cost carries in a crew's bill. + Volume float64 +} + +// DefaultShapes returns the three seat shapes crews are picked for. The +// worker takes half its quality from the agentic index and reads barely +// anything but cached input; the high seat judges the three indexes evenly, +// needs tools and images both, and sees the same token mix on a sliver of the +// volume; the mastermind reads intelligence and agentic only, on a terse +// ten-in-one mix with no cache and a fraction of the volume. +func DefaultShapes() map[Seat]SeatShape { + return map[Seat]SeatShape{ + Worker: { + Weights: [3]float64{0.2, 0.3, 0.5}, + InOut: 100, + CacheShare: 0.75, + NeedTools: true, + MinContext: 200_000, + Volume: 0.90, + }, + High: { + Weights: [3]float64{1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0}, + InOut: 100, + CacheShare: 0.75, + NeedTools: true, + NeedImages: true, + MinContext: 200_000, + Volume: 0.08, + }, + Mastermind: { + Weights: [3]float64{0.6, 0, 0.4}, + InOut: 10, + MinContext: 200_000, + Volume: 0.02, + }, + } +} + +// A Family is which candidates a crew may be drawn from. +type Family int + +const ( + // Open draws only from candidates marked Open. + Open Family = iota + // All draws from every candidate. + All +) + +// A Crew is one (worker, high, mastermind) pick. Bill is the expected dollars +// per 1M task tokens once each seat's share of the task is paid for at its +// seat cost; Quality is the mean of the three seat qualities on a 0-100 +// scale. +type Crew struct { + Worker string + High string + Mastermind string + Bill float64 + Quality float64 + // Estimated marks, for the model in each seat, which of its indexes + // were scored by estimate rather than measurement: the row is the seat + // and the column the index. A crew picked on measured indexes alone is + // all false. + Estimated [3][3]bool + // Measured marks, for the model in each seat, whether a pool's own + // rating for that model entered the seat's quality: the row is the seat + // and the value says the seat's model carried a rating with a positive + // observation count. A crew picked with no prior, or whose seats the + // prior never rated, is all false. + Measured [3]bool +} + +// Floor is the share of a seat's best quality in the family that a candidate +// must reach to stay in the running for that seat. A model that cannot do +// the job is not cheap, whatever it costs. +const Floor = 0.80 + +// PriorWeightAt is the observation count at which a pool's own rating for a +// model carries the same weight in a seat's quality as the catalog's +// published indexes: at N observations the rating is worth half the seat and +// the catalog the other half. A rating with no observations never enters. +const PriorWeightAt = 30 + +// A Rating is a pool's measured quality for one model on one seat, on the +// same 0-100 scale quality is read on, with the number of observations behind +// it. +type Rating struct { + Mean float64 + N int +} + +// A Prior is a pool's measured quality per seat, keyed by canonical model id: +// the rating a seat's quality is blended towards, by how many observations +// back it. A nil Prior is no prior at all, and every seat keeps the catalog +// quality. +type Prior map[Seat]map[string]Rating + +// A Cell is one measurement a caller hands PriorFromCells: a role, a model id +// and the pool's mean quality with the observations behind it. It is the shape +// a measurement document reads into, so crewpick need not import the reader. +type Cell struct { + Role string + Model string + Mean float64 + N int +} + +// seatWords names the roles a Cell's role word maps to a seat by, folded to +// one spelling so a document's own case never matters. Any other word names +// no seat and is ignored. +var seatWords = map[string]Seat{ + "worker": Worker, + "high": High, + "mastermind": Mastermind, +} + +// PriorFromCells reads a pool's measurements into a Prior: a cell names a +// seat by its role word, a model by an id that canonical resolves to the +// canonical id a candidate is looked up by, and carries a mean and a count. +// A cell whose role names no seat, or whose count is below minInstalls, is +// ignored, and so is a count of zero or less. A nil canonical leaves the id +// as it stands. When no cell survives the prior is nil, which is the same as +// no prior at all. +func PriorFromCells(cells []Cell, minInstalls int, canonical func(string) string) Prior { + var prior Prior + for _, c := range cells { + seat, ok := seatWords[strings.ToLower(strings.TrimSpace(c.Role))] + if !ok || c.N < minInstalls || c.N <= 0 { + continue + } + model := c.Model + if canonical != nil { + model = canonical(model) + } + if prior == nil { + prior = Prior{} + } + if prior[seat] == nil { + prior[seat] = map[string]Rating{} + } + prior[seat][model] = Rating{Mean: c.Mean, N: c.N} + } + return prior +} + +// MergePriors folds two priors into one. Seat by seat, a rating both hold for +// a model is combined by observation count — the two means weighted by the +// counts behind them, the counts added — and a rating either holds alone is +// carried whole. A nil prior on either side is the other, and two nils are +// nil. The priors given are read and never changed, so the answer is never +// one of them. +func MergePriors(a, b Prior) Prior { + if a == nil { + return b + } + if b == nil { + return a + } + merged := make(Prior) + for seat, ratings := range a { + for model, r := range ratings { + if merged[seat] == nil { + merged[seat] = map[string]Rating{} + } + merged[seat][model] = r + } + } + for seat, ratings := range b { + for model, r := range ratings { + got, ok := merged[seat][model] + if !ok { + if merged[seat] == nil { + merged[seat] = map[string]Rating{} + } + merged[seat][model] = r + continue + } + n := got.N + r.N + if n <= 0 { + merged[seat][model] = Rating{} + continue + } + merged[seat][model] = Rating{ + Mean: (float64(got.N)*got.Mean + float64(r.N)*r.Mean) / float64(n), + N: n, + } + } + } + return merged +} + +// SeatCost is the seat's expected cost of a candidate, in dollars per 1M +// input-equivalent tokens: the prompt price blended with the cache-read +// price by the seat's cache share, plus the completion price spread over the +// seat's input-to-output ratio. A candidate that publishes no cache-read +// price pays the prompt price on the cache share too. +func SeatCost(c Candidate, shape SeatShape) float64 { + cache := c.PromptPrice + if c.HasCacheRead { + cache = c.CacheReadPrice + } + return (1-shape.CacheShare)*c.PromptPrice + shape.CacheShare*cache + c.CompletionPrice/shape.InOut +} + +// SeatQuality is the candidate's quality for the seat on a 0-100 scale: each +// index counted against the pool's best for that index and weighted by the +// seat's weights. The pool sets the scale, so the same candidate scores +// differently in a stronger pool. A missing index is estimated from the +// pool's donors exactly as in Front. A row that is not a candidate at all, +// and any row against a pool with no candidates to raise a scale, score +// zero. +func SeatQuality(c Candidate, shape SeatShape, pool []Candidate) float64 { + return SeatQualityWith(c, shape, pool, Worker, nil) +} + +// SeatQualityWith is SeatQuality with the pool's prior blended in for the seat +// given: a rating the prior holds for the candidate's canonical id, with a +// positive observation count, moves the seat's quality towards the rating's +// mean by N/(N+PriorWeightAt), and a seat the prior never rates — or rates +// with no observations — keeps the catalog quality unchanged. +func SeatQualityWith(c Candidate, shape SeatShape, pool []Candidate, seat Seat, prior Prior) float64 { + if !isCandidate(c) { + return 0 + } + scale := maxima(pool) + r := fill(c, readTally(pool), scale) + q, _ := blendQuality(c.ID, quality(r.idx, shape, scale), seat, prior) + return q +} + +// Front returns the pareto front of (bill, quality) over every crew the +// candidates can field in the family, sorted by bill: a crew on it is beaten +// by no other crew on both, and every crew off it is beaten by one on it. +// +// shapes must carry an entry for every seat. Per seat the running is +// narrowed twice before crews are enumerated — family and the seat's needs +// first, then Floor against that seat's best in the family — and a crew is +// refused while its high seat comes from the worker's own vendor, so the two +// seats that see the same work never share one shop. Ties in (bill, quality) +// break to the crew scored on fewer estimated indexes, and ties past that by +// id order — worker, then high, then mastermind — which is what makes the +// front a property of the candidates rather than of their order. +func Front(candidates []Candidate, shapes map[Seat]SeatShape, fam Family) []Crew { + return FrontWith(candidates, shapes, fam, nil) +} + +// FrontWith is Front with the pool's prior blended into each seat's quality: a +// rating the prior holds for a seat's model, with a positive observation +// count, moves that seat's quality towards the rating's mean by +// N/(N+PriorWeightAt), the seat's measured flag says so, and the floor and the +// crew's quality read the blended value. A nil prior leaves every seat on the +// catalog quality, exactly as Front does. The front is a property of the +// candidates and the prior, never of the order they arrive in. +func FrontWith(candidates []Candidate, shapes map[Seat]SeatShape, fam Family, prior Prior) []Crew { + scale := maxima(candidates) + t := readTally(candidates) + readings := make([]reading, 0, len(candidates)) + for _, c := range candidates { + readings = append(readings, fill(c, t, scale)) + } + workers := shortlist(readings, shapes[Worker], fam, scale, Worker, prior) + highs := shortlist(readings, shapes[High], fam, scale, High, prior) + minds := shortlist(readings, shapes[Mastermind], fam, scale, Mastermind, prior) + vw := shapes[Worker].Volume + vh := shapes[High].Volume + vm := shapes[Mastermind].Volume + + crews := make([]Crew, 0, len(workers)*len(highs)*len(minds)) + for _, w := range workers { + for _, h := range highs { + if vendor(w.c.ID) == vendor(h.c.ID) { + continue + } + for _, m := range minds { + crews = append(crews, Crew{ + Worker: w.c.ID, + High: h.c.ID, + Mastermind: m.c.ID, + Bill: vw*w.cost + vh*h.cost + vm*m.cost, + Quality: (w.quality + h.quality + m.quality) / 3, + Estimated: [3][3]bool{w.est, h.est, m.est}, + Measured: [3]bool{w.measured, h.measured, m.measured}, + }) + } + } + } + + // The front: cheapest bill first, best quality first within a bill, and + // a crew survives when its quality beats everything cheaper so far. + sort.SliceStable(crews, func(i, j int) bool { + if crews[i].Bill != crews[j].Bill { + return crews[i].Bill < crews[j].Bill + } + if crews[i].Quality != crews[j].Quality { + return crews[i].Quality > crews[j].Quality + } + return estimateCount(crews[i]) < estimateCount(crews[j]) + }) + var front []Crew + best := math.Inf(-1) + for _, crew := range crews { + if crew.Quality > best+1e-9 { + front = append(front, crew) + best = crew.Quality + } + } + return front +} + +// Presets reads the three named picks off a front: frugal is the cheapest +// crew, max the dearest, and balanced the knee between them — the crew +// farthest above the straight line from one end of the front to the other in +// (ln bill, quality), which is where money stops buying quality in earnest. +// A front of one crew is all three; an empty front is none. +func Presets(front []Crew) (frugal, balanced, max Crew) { + if len(front) == 0 { + return Crew{}, Crew{}, Crew{} + } + frugal, balanced, max = front[0], knee(front), front[len(front)-1] + return frugal, balanced, max +} + +// AtKnob reads one pick off the front at knob k in [0, 1]: the budget runs +// geometrically from the frugal bill to the max bill — lo*(hi/lo)^k — and +// the pick is the best-quality crew whose bill fits it, so k of 0 is frugal +// and k of 1 is max. An empty front picks nothing. +func AtKnob(front []Crew, k float64) Crew { + if len(front) == 0 { + return Crew{} + } + lo, hi := front[0].Bill, front[len(front)-1].Bill + budget := lo * math.Pow(hi/lo, k) + var pick Crew + for _, crew := range front { + if crew.Bill <= budget*(1+1e-9) { + pick = crew + } + } + return pick +} + +// knee returns the front's knee: the crew farthest above the straight line +// from the first crew to the last in (ln bill, quality). Ties go to the +// earlier crew, and a front whose two ends share a bill has no line to be +// above — its best crew carries the name. +func knee(front []Crew) Crew { + x0, y0 := math.Log(front[0].Bill), front[0].Quality + x1, y1 := math.Log(front[len(front)-1].Bill), front[len(front)-1].Quality + if x1 == x0 { + return front[len(front)-1] + } + pick := front[0] + best := math.Inf(-1) + for _, crew := range front { + d := (crew.Quality - y0) - (y1-y0)*(math.Log(crew.Bill)-x0)/(x1-x0) + if d > best { + pick, best = crew, d + } + } + return pick +} + +// A seatPick is a candidate already priced and scored for one seat, on the +// candidate's reading: measured and estimated indexes alike. measured says +// whether the pool's own rating for the seat entered the quality. +type seatPick struct { + c Candidate + idx [3]float64 + est [3]bool + cost float64 + quality float64 + measured bool +} + +// shortlist returns a seat's running: the candidates that pass family and the +// seat's needs, kept when their quality reaches Floor of the family's best +// for the seat, in id order. The seat's quality is read with the pool's prior +// blended in, and measured marks the picks the prior touched. +func shortlist(readings []reading, shape SeatShape, fam Family, scale [3]float64, seat Seat, prior Prior) []seatPick { + var running []seatPick + top := 0.0 + for _, r := range readings { + if !eligible(r.c, shape, fam) { + continue + } + q, measured := blendQuality(r.c.ID, quality(r.idx, shape, scale), seat, prior) + pick := seatPick{c: r.c, idx: r.idx, est: r.est, cost: SeatCost(r.c, shape), quality: q, measured: measured} + running = append(running, pick) + top = math.Max(top, pick.quality) + } + var kept []seatPick + for _, pick := range running { + if pick.quality >= Floor*top { + kept = append(kept, pick) + } + } + sort.Slice(kept, func(i, j int) bool { return kept[i].c.ID < kept[j].c.ID }) + return kept +} + +// quality is SeatQuality against pre-read maxima, on a candidate's reading. +func quality(idx [3]float64, shape SeatShape, scale [3]float64) float64 { + if scale[Intelligence] <= 0 || scale[Coding] <= 0 || scale[Agentic] <= 0 { + return 0 + } + w := shape.Weights + return 100 * (w[Intelligence]*idx[Intelligence]/scale[Intelligence] + + w[Coding]*idx[Coding]/scale[Coding] + + w[Agentic]*idx[Agentic]/scale[Agentic]) +} + +// blendQuality blends the pool's prior for one seat into the catalog quality: +// a rating the prior holds for the model on the seat, with a positive +// observation count, carries weight N/(N+PriorWeightAt) against the catalog's +// own quality, and the second result reports whether the rating entered. A +// seat the prior never rates, or rates with no observations, keeps the +// catalog quality and reads measured false. +func blendQuality(model string, catalog float64, seat Seat, prior Prior) (float64, bool) { + if prior == nil { + return catalog, false + } + rating, ok := prior[seat][model] + if !ok || rating.N <= 0 { + return catalog, false + } + w := float64(rating.N) / float64(rating.N+PriorWeightAt) + return (1-w)*catalog + w*rating.Mean, true +} + +// maxima reads the pool's largest measured value of each index — the scale +// quality is measured against and the ceiling an estimate never crosses. +// Only real candidates raise a ceiling, and a row that publishes only some +// of its indexes raises only those. +func maxima(pool []Candidate) (scale [3]float64) { + for _, c := range pool { + if !isCandidate(c) { + continue + } + v := measured(c) + for i, val := range v { + scale[i] = math.Max(scale[i], val) + } + } + return scale +} + +// measured is the candidate's three indexes as it published them: an +// unpublished index reads as zero. +func measured(c Candidate) [3]float64 { + return [3]float64{c.Intelligence, c.Coding, c.Agentic} +} + +// A tally is one call's donor statistics: for every ordered pair of indexes, +// how many candidates — the donors — carry both measured, and the median of +// the first over the second among them. A missing index is estimated from +// these. +type tally struct { + count [3][3]int + median [3][3]float64 +} + +// readTally reads the pool's tally; the donors of a pair of indexes are the +// candidates carrying both measured. +func readTally(pool []Candidate) tally { + var ratios [3][3][]float64 + for _, c := range pool { + if !isCandidate(c) { + continue + } + v := measured(c) + for _, x := range [3]Index{Intelligence, Coding, Agentic} { + for _, y := range [3]Index{Intelligence, Coding, Agentic} { + if x != y && v[x] > 0 && v[y] > 0 { + ratios[x][y] = append(ratios[x][y], v[x]/v[y]) + } + } + } + } + var t tally + for _, x := range [3]Index{Intelligence, Coding, Agentic} { + for _, y := range [3]Index{Intelligence, Coding, Agentic} { + t.count[x][y] = len(ratios[x][y]) + if len(ratios[x][y]) >= 3 { + t.median[x][y] = medianOf(ratios[x][y]) + } + } + } + return t +} + +// medianOf is the middle of the values in order, or the mean of the two +// middles when there is no single middle. +func medianOf(values []float64) float64 { + slices.Sort(values) + n := len(values) + if n%2 == 1 { + return values[n/2] + } + return (values[n/2-1] + values[n/2]) / 2 +} + +// A reading is a candidate's three indexes ready for scoring — the measured +// ones as published, the missing ones estimated from the call's donors — +// with est remembering which of the three were estimated rather than +// measured. +type reading struct { + c Candidate + idx [3]float64 + est [3]bool +} + +// fill reads the candidate's three indexes for scoring. A measured index is +// taken as published; a missing one is estimated from the ones present: each +// present index Y offers Y times the tally's median of the missing index over +// Y among the donors carrying both — or, when fewer than three donors carry +// the pair, equal standing, the missing index read where its present one +// stands, X over the pool's largest X equal to Y over the pool's largest Y — +// and the mean of the offers is kept, never above the pool's largest measured +// value of the missing index. A row that is no candidate at all fills to +// zeros. +func fill(c Candidate, t tally, scale [3]float64) reading { + var r reading + if !isCandidate(c) { + return r + } + r.c = c + v := measured(c) + r.idx = v + for _, x := range [3]Index{Intelligence, Coding, Agentic} { + if v[x] > 0 { + continue + } + var offers []float64 + for _, y := range [3]Index{Intelligence, Coding, Agentic} { + if y == x || v[y] <= 0 { + continue + } + if t.count[x][y] >= 3 { + offers = append(offers, v[y]*t.median[x][y]) + } else { + offers = append(offers, v[y]/scale[y]*scale[x]) + } + } + sum := 0.0 + for _, offer := range offers { + sum += offer + } + r.idx[x] = math.Min(sum/float64(len(offers)), scale[x]) + r.est[x] = true + } + return r +} + +// estimateCount reads how many of a crew's picks were scored on estimated +// indexes — the tiebreak between crews that tie on bill and quality. +func estimateCount(c Crew) int { + n := 0 + for _, seat := range c.Estimated { + for _, est := range seat { + if est { + n++ + } + } + } + return n +} + +// isCandidate reports whether a row is a candidate at all: at least one of +// the three indexes present and a prompt price above zero. Everything +// downstream — eligibility, the floor, the maxima — starts from this. +func isCandidate(c Candidate) bool { + return (c.Intelligence > 0 || c.Coding > 0 || c.Agentic > 0) && c.PromptPrice > 0 +} + +// eligible reports whether a candidate may sit the seat: family first, then +// what the seat needs the model to take and call, then the window it needs. +func eligible(c Candidate, shape SeatShape, fam Family) bool { + if !isCandidate(c) { + return false + } + if fam == Open && !c.Open { + return false + } + if shape.NeedTools && !c.Tools { + return false + } + if shape.NeedImages && !c.Images { + return false + } + return c.Context >= shape.MinContext +} + +// vendor is the id text before the first slash, or the whole id when there is +// no slash. +func vendor(id string) string { + if i := strings.Index(id, "/"); i >= 0 { + return id[:i] + } + return id +} diff --git a/internal/crewpick/crewpick_test.go b/internal/crewpick/crewpick_test.go new file mode 100644 index 0000000000..f884530002 --- /dev/null +++ b/internal/crewpick/crewpick_test.go @@ -0,0 +1,833 @@ +package crewpick + +import ( + "encoding/json" + "fmt" + "math" + "os" + "slices" + "testing" +) + +// A fixture row in its own field names: a null index reads as zero — +// missing, the way the package reads it — and a null cache-read price reads +// as unpublished. The partially published row and the rows with unpublished +// cache prices stay in the fixture, so the estimate and exclusion rules +// have something to bite on. +type row struct { + ID string `json:"id"` + Open bool `json:"open"` + I float64 `json:"I"` + C float64 `json:"C"` + A float64 `json:"A"` + Pi float64 `json:"pi"` + Po float64 `json:"po"` + Cr *float64 `json:"cr"` + Ctx int `json:"ctx"` + Img bool `json:"img"` + Tools bool `json:"tools"` +} + +// loadCandidates reads the fixture into candidates. +func loadCandidates(t *testing.T) []Candidate { + t.Helper() + raw, err := os.ReadFile("testdata/candidates.json") + if err != nil { + t.Fatal(err) + } + var rows []row + if err := json.Unmarshal(raw, &rows); err != nil { + t.Fatal(err) + } + cands := make([]Candidate, 0, len(rows)) + for _, r := range rows { + c := Candidate{ + ID: r.ID, + Open: r.Open, + Intelligence: r.I, + Coding: r.C, + Agentic: r.A, + PromptPrice: r.Pi, + CompletionPrice: r.Po, + Context: r.Ctx, + Images: r.Img, + Tools: r.Tools, + } + if r.Cr != nil { + c.CacheReadPrice = *r.Cr + c.HasCacheRead = true + } + cands = append(cands, c) + } + return cands +} + +// model builds a candidate that can sit any seat: every index at 100, tools +// and images, a 200k window, and a published cache price. +func model(id string) Candidate { + return Candidate{ + ID: id, + Open: true, + Intelligence: 100, + Coding: 100, + Agentic: 100, + PromptPrice: 1, + CompletionPrice: 4, + CacheReadPrice: 0.1, + HasCacheRead: true, + Context: 200_000, + Images: true, + Tools: true, + } +} + +// donors builds n fully measured candidates that can sit no seat — a window +// too small for any of them — so they feed the estimates and the ceilings +// and nothing else. +func donors(prefix string, n int) []Candidate { + rows := make([]Candidate, 0, n) + for i := range n { + d := model(fmt.Sprintf("%s/d%d", prefix, i)) + d.Context = 1000 + rows = append(rows, d) + } + return rows +} + +// wantCrew checks a pick against its worker|high|mastermind line. +func wantCrew(t *testing.T, name string, got Crew, want string) { + t.Helper() + line := got.Worker + "|" + got.High + "|" + got.Mastermind + if line != want { + t.Fatalf("%s = %s, want %s", name, line, want) + } +} + +// The fixture's front is a fixed property of its rows: seven crews in the +// open family, eighteen in all, and these are the picks each front reads. +// The all-family balanced crew is where money stops buying quality — a bill +// of 0.590 at quality 95.4. +func TestPresetsReadTheFixtureFront(t *testing.T) { + cands := loadCandidates(t) + shapes := DefaultShapes() + + open := Front(cands, shapes, Open) + if len(open) != 7 { + t.Fatalf("open front holds %d crews, want 7", len(open)) + } + frugal, balanced, max := Presets(open) + wantCrew(t, "open frugal", frugal, "deepseek/deepseek-v4-flash-0731|z-ai/glm-5.3-flash|z-ai/glm-5.3-flash") + wantCrew(t, "open balanced", balanced, "z-ai/glm-5.3-flash|moonshotai/kimi-k3|z-ai/glm-5.3") + wantCrew(t, "open max", max, "z-ai/glm-5.3|moonshotai/kimi-k3|z-ai/glm-5.3") + + all := Front(cands, shapes, All) + if len(all) != 18 { + t.Fatalf("all front holds %d crews, want 18", len(all)) + } + frugal, balanced, max = Presets(all) + wantCrew(t, "all frugal", frugal, "z-ai/glm-5.3-flash|google/gemini-3.8-flash|z-ai/glm-5.3-flash") + wantCrew(t, "all balanced", balanced, "z-ai/glm-5.3-flash|anthropic/claude-fable-5.1|anthropic/claude-fable-5.1") + wantCrew(t, "all max", max, "anthropic/claude-fable-5.1|openai/gpt-6-astra|anthropic/claude-fable-5.1") + if math.Abs(balanced.Bill-0.590) > 0.005 { + t.Fatalf("all balanced bill = %f, want about 0.590", balanced.Bill) + } + if math.Abs(balanced.Quality-95.4) > 0.05 { + t.Fatalf("all balanced quality = %f, want about 95.4", balanced.Quality) + } +} + +// The high seat sees the same work the worker does, so a crew never draws it +// from the worker's own vendor. Every crew on both fixture fronts holds +// that line, and a pool that cannot field two vendors fields no crew at all. +func TestTheHighSeatNeverSharesAVendorWithTheWorker(t *testing.T) { + cands := loadCandidates(t) + for _, fam := range []Family{Open, All} { + for _, crew := range Front(cands, DefaultShapes(), fam) { + if vendor(crew.Worker) == vendor(crew.High) { + t.Fatalf("worker %s and high %s share a vendor", crew.Worker, crew.High) + } + } + } + + oneVendor := []Candidate{model("one/first"), model("one/second")} + if front := Front(oneVendor, DefaultShapes(), All); len(front) != 0 { + t.Fatalf("a one-vendor pool fielded %d crews, want none", len(front)) + } +} + +// The floor keeps a seat's running to candidates that can do the job: a +// model whose quality falls below Floor of the seat's best in the family is +// not cheap and never reaches a crew. The boundary is inclusive — a +// candidate sitting exactly on the floor runs, and at a tenth of the price +// of anything above it, it runs first. +func TestTheFloorDropsAModelThatCannotDoTheJob(t *testing.T) { + pool := []Candidate{model("a/m"), model("c/h")} + + // Cheap enough to be every crew's frugal worker, and short one index: + // 70 of the worker seat's 100, 76 of the mastermind's, and ineligible + // for the high seat, which needs images. + cheap := model("b/w") + cheap.Agentic = 40 + cheap.Images = false + cheap.PromptPrice = 0.01 + cheap.CompletionPrice = 0.02 + cheap.HasCacheRead = false + pool = append(pool, cheap) + + // Exactly on the worker seat's floor: 80 of 100. + edge := model("e/b") + edge.Agentic = 60 + edge.PromptPrice = 0.02 + edge.CompletionPrice = 0.04 + edge.HasCacheRead = false + pool = append(pool, edge) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + frugal, _, _ := Presets(front) + if frugal.Worker != "e/b" { + t.Fatalf("frugal worker = %s, want the on-floor candidate e/b", frugal.Worker) + } + for _, crew := range front { + for _, id := range []string{crew.Worker, crew.High, crew.Mastermind} { + if id == "b/w" { + t.Fatalf("the below-floor model b/w reached a crew: %+v", crew) + } + } + } +} + +// A row carrying none of its three indexes, or priced at zero, is not a +// candidate at all: it never reaches a seat and it raises no ceiling. Both +// rules bite where the floor cannot — a free model would clear any floor, +// and a bare row carries no index to estimate from. +func TestARowWithNoIndexesAtAllOrNoPriceIsNoCandidate(t *testing.T) { + good := []Candidate{model("a/m"), model("c/h")} + good[1].Intelligence = 90 + good[1].PromptPrice = 2 + good[1].CompletionPrice = 8 + good[1].CacheReadPrice = 0.2 + + bare := model("d/x") + bare.Intelligence = 0 + bare.Coding = 0 + bare.Agentic = 0 + bare.PromptPrice = 0.01 + bare.CompletionPrice = 0.02 + bare.HasCacheRead = false + + free := model("e/y") + free.PromptPrice = 0 + + pool := append(slices.Clone(good), bare, free) + front := Front(pool, DefaultShapes(), All) + for _, crew := range front { + for _, id := range []string{crew.Worker, crew.High, crew.Mastermind} { + if id == "d/x" || id == "e/y" { + t.Fatalf("a non-candidate row reached a crew: %+v", crew) + } + } + } + // The non-candidate rows leave the real candidates' front exactly what + // it was without them. + if want := Front(good, DefaultShapes(), All); !slices.Equal(front, want) { + t.Fatalf("the non-candidate rows moved the front:\n got %+v\nwant %+v", front, want) + } +} + +// A model publishing only its intelligence index — a release scored in +// stages — is scored on it, with the coding and agentic indexes estimated +// from the donors, and at a hundredth of the price it takes the worker seat. +func TestAModelWithOnlyAnIntelligenceIndexIsChosen(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h")} + newRelease := model("b/x") + newRelease.Coding = 0 + newRelease.Agentic = 0 + newRelease.PromptPrice = 0.01 + newRelease.CompletionPrice = 0.02 + newRelease.HasCacheRead = false + pool = append(pool, newRelease) + pool = append(pool, donors("v", 3)...) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + frugal, _, _ := Presets(front) + if frugal.Worker != "b/x" { + t.Fatalf("frugal worker = %s, want the partially scored b/x", frugal.Worker) + } + want := [3][3]bool{{false, true, true}, {}, {false, true, true}} + if frugal.Estimated != want { + t.Fatalf("the b/x crew's Estimated = %v, want %v", frugal.Estimated, want) + } +} + +// A model publishing only its coding index — many an older model — is +// scored on it, with the intelligence and agentic indexes estimated, and at +// a hundredth of the price it takes the worker seat too. +func TestAModelWithOnlyACodingIndexIsChosen(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h")} + older := model("b/x") + older.Intelligence = 0 + older.Agentic = 0 + older.PromptPrice = 0.01 + older.CompletionPrice = 0.02 + older.HasCacheRead = false + pool = append(pool, older) + pool = append(pool, donors("v", 3)...) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + frugal, _, _ := Presets(front) + if frugal.Worker != "b/x" { + t.Fatalf("frugal worker = %s, want the partially scored b/x", frugal.Worker) + } + want := [3][3]bool{{true, false, true}, {}, {true, false, true}} + if frugal.Estimated != want { + t.Fatalf("the b/x crew's Estimated = %v, want %v", frugal.Estimated, want) + } +} + +// The donor estimate is the median of the missing index over the present +// one among the donors carrying both — not the mean, which the far-out +// fourth donor drags away — and the rows missing either index of the pair +// never donate. The row missing its intelligence index still raises the +// coding ceiling with its own 400. +func TestTheDonorEstimateIsTheMedianRatio(t *testing.T) { + shape := SeatShape{Weights: [3]float64{0, 1, 0}} + pool := []Candidate{ + {ID: "a/d1", Intelligence: 10, Coding: 10, Agentic: 10, PromptPrice: 1}, + {ID: "a/d2", Intelligence: 10, Coding: 20, Agentic: 10, PromptPrice: 1}, + {ID: "a/d3", Intelligence: 10, Coding: 30, Agentic: 10, PromptPrice: 1}, + {ID: "a/d4", Intelligence: 10, Coding: 300, Agentic: 10, PromptPrice: 1}, + {ID: "b/noc", Intelligence: 10, Agentic: 10, PromptPrice: 1}, + {ID: "b/noi", Coding: 400, Agentic: 10, PromptPrice: 1}, + } + x := Candidate{ID: "c/x", Intelligence: 10, PromptPrice: 1} + + // The donors' coding over intelligence ratios run 1, 2, 3, 30: the + // median reads 2.5 and the estimate 25 of a 400 ceiling, where the mean + // would read 9 and the estimate 90. + if got := SeatQuality(x, shape, pool); got != 6.25 { + t.Fatalf("SeatQuality on the median estimate = %v, want 6.25", got) + } +} + +// With fewer than three donors carrying a pair, the estimate falls back to +// equal standing — the missing index read at the present one's share of +// the pool's ceilings — not to the median of two donors, and not to +// exclusion. +func TestFewerThanThreeDonorsEstimatesByEqualStanding(t *testing.T) { + shape := SeatShape{Weights: [3]float64{0, 1, 0}} + pool := []Candidate{ + {ID: "a/d1", Intelligence: 100, Coding: 50, Agentic: 10, PromptPrice: 1}, + {ID: "a/d2", Intelligence: 50, Coding: 100, Agentic: 10, PromptPrice: 1}, + } + x := Candidate{ID: "b/x", Intelligence: 80, PromptPrice: 1} + + // Equal standing reads x's coding at x's share of the intelligence + // ceiling, 80 of 100; the two donors' median would read 100. + if got := SeatQuality(x, shape, pool); got != 80 { + t.Fatalf("SeatQuality on equal standing = %v, want 80", got) + } +} + +// An estimate never crosses the pool's largest measured value of its +// index: the donors' coding over intelligence median runs 20, and 50 times +// that would read 1000 of a 300 ceiling. +func TestAnEstimateIsCappedAtTheLargestMeasuredIndex(t *testing.T) { + shape := SeatShape{Weights: [3]float64{0, 1, 0}} + pool := []Candidate{ + {ID: "a/d1", Intelligence: 10, Coding: 100, Agentic: 10, PromptPrice: 1}, + {ID: "a/d2", Intelligence: 10, Coding: 200, Agentic: 10, PromptPrice: 1}, + {ID: "a/d3", Intelligence: 10, Coding: 300, Agentic: 10, PromptPrice: 1}, + } + x := Candidate{ID: "b/x", Intelligence: 50, PromptPrice: 1} + + if got := SeatQuality(x, shape, pool); got != 100 { + t.Fatalf("SeatQuality on the capped estimate = %v, want 100", got) + } +} + +// The estimate is never silent: the crew says, for the model in each seat, +// which of its three indexes were scored by estimate and which measured. +// The worker publishes only its intelligence index and the mastermind only +// its coding one; the high seat is measured throughout. +func TestTheResultMarksEstimatedIndexes(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h")} + worker := model("b/x") + worker.Coding = 0 + worker.Agentic = 0 + worker.Images = false + worker.PromptPrice = 0.01 + worker.CompletionPrice = 0.02 + worker.HasCacheRead = false + mind := model("d/m") + mind.Intelligence = 0 + mind.Agentic = 0 + mind.Tools = false + mind.PromptPrice = 0.005 + mind.CompletionPrice = 0.01 + mind.HasCacheRead = false + pool = append(pool, worker, mind) + pool = append(pool, donors("v", 3)...) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + frugal, _, _ := Presets(front) + if frugal.Worker != "b/x" || frugal.Mastermind != "d/m" { + t.Fatalf("frugal = %s|%s|%s, want b/x on the worker seat and d/m on the mastermind's", frugal.Worker, frugal.High, frugal.Mastermind) + } + want := [3][3]bool{{false, true, true}, {}, {true, false, true}} + if frugal.Estimated != want { + t.Fatalf("the crew's Estimated = %v, want %v", frugal.Estimated, want) + } +} + +// Two candidates for a seat otherwise tied on quality and cost run to the +// one with fewer estimated indexes — even when its id reads later, and +// whichever way round the pool arrives. +func TestATieOnQualityAndCostRunsToFewerEstimatedIndexes(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h")} + // Twins for the mastermind seat, identical in every number the picker + // reads, but aaa runs on two estimated indexes and zzz on none. + aaa := model("aaa/two") + aaa.Intelligence = 0 + aaa.Coding = 80 + aaa.Agentic = 0 + aaa.Tools = false + aaa.Images = false + aaa.PromptPrice = 0.1 + aaa.CompletionPrice = 0.2 + aaa.HasCacheRead = false + zzz := aaa + zzz.ID = "zzz/one" + zzz.Intelligence = 80 + zzz.Agentic = 80 + pool = append(pool, aaa, zzz) + pool = append(pool, donors("v", 3)...) + + front := Front(pool, DefaultShapes(), All) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + if front[0].Mastermind != "zzz/one" { + t.Fatalf("a tie went to the estimated twin: %+v", front[0]) + } + for _, crew := range front { + if crew.Mastermind == "aaa/two" { + t.Fatalf("the estimated twin survived its tie: %+v", crew) + } + } + + reversed := slices.Clone(pool) + slices.Reverse(reversed) + if two := Front(reversed, DefaultShapes(), All); !slices.Equal(front, two) { + t.Fatal("the tie broke differently when the pool arrived reversed") + } +} + +// Estimates, caps and ties read the same whatever order the candidates +// arrive in: a pool of partially published models gives the same front +// shuffled as it does in hand order. +func TestPartialCandidatesGiveTheSameFrontInAnyOrder(t *testing.T) { + newRelease := model("b/new") + newRelease.Coding = 0 + newRelease.Agentic = 0 + newRelease.PromptPrice = 0.01 + newRelease.CompletionPrice = 0.02 + newRelease.HasCacheRead = false + older := model("d/old") + older.Intelligence = 0 + older.Agentic = 0 + older.Tools = false + staged := model("e/staged") + staged.Agentic = 0 + pool := []Candidate{model("a/w"), model("c/h"), newRelease, older, staged} + pool = append(pool, donors("v", 3)...) + + one := Front(pool, DefaultShapes(), All) + if len(one) == 0 { + t.Fatal("the pool must field crews") + } + if one[0].Estimated == ([3][3]bool{}) { + t.Fatal("the cheapest crew carries no estimated index; the partial candidates never reached it") + } + reversed := slices.Clone(pool) + slices.Reverse(reversed) + if two := Front(reversed, DefaultShapes(), All); !slices.Equal(one, two) { + t.Fatal("the front changed when the partial candidates arrived reversed") + } + rotated := append(slices.Clone(pool[3:]), pool[:3]...) + if three := Front(rotated, DefaultShapes(), All); !slices.Equal(one, three) { + t.Fatal("the front changed when the pool arrived rotated") + } +} + +// An input with no index missing picks on measurements alone: the fixture's +// partially published row left out, both fronts hold the same sizes and +// the same frugal, balanced and max picks the whole fixture holds, and no +// estimated flag is set anywhere — the partial row never raised a ceiling, +// so leaving it out moves nothing measured. +func TestNothingMissingLeavesEveryPickUnchanged(t *testing.T) { + cands := loadCandidates(t) + var full []Candidate + for _, c := range cands { + if c.Intelligence > 0 && c.Coding > 0 && c.Agentic > 0 { + full = append(full, c) + } + } + shapes := DefaultShapes() + + open := Front(full, shapes, Open) + if len(open) != 7 { + t.Fatalf("the full open front holds %d crews, want 7", len(open)) + } + frugal, balanced, max := Presets(open) + wantCrew(t, "open frugal", frugal, "deepseek/deepseek-v4-flash-0731|z-ai/glm-5.3-flash|z-ai/glm-5.3-flash") + wantCrew(t, "open balanced", balanced, "z-ai/glm-5.3-flash|moonshotai/kimi-k3|z-ai/glm-5.3") + wantCrew(t, "open max", max, "z-ai/glm-5.3|moonshotai/kimi-k3|z-ai/glm-5.3") + + all := Front(full, shapes, All) + if len(all) != 18 { + t.Fatalf("the full all front holds %d crews, want 18", len(all)) + } + frugal, balanced, max = Presets(all) + wantCrew(t, "all frugal", frugal, "z-ai/glm-5.3-flash|google/gemini-3.8-flash|z-ai/glm-5.3-flash") + wantCrew(t, "all balanced", balanced, "z-ai/glm-5.3-flash|anthropic/claude-fable-5.1|anthropic/claude-fable-5.1") + wantCrew(t, "all max", max, "anthropic/claude-fable-5.1|openai/gpt-6-astra|anthropic/claude-fable-5.1") + if math.Abs(balanced.Bill-0.590) > 0.005 { + t.Fatalf("all balanced bill = %f, want about 0.590", balanced.Bill) + } + if math.Abs(balanced.Quality-95.4) > 0.05 { + t.Fatalf("all balanced quality = %f, want about 95.4", balanced.Quality) + } + for _, front := range [][]Crew{open, all} { + for _, crew := range front { + if crew.Estimated != ([3][3]bool{}) { + t.Fatalf("a fully measured pool raised an estimated flag: %+v", crew) + } + } + } +} + +// The knob runs from one end of the front to the other: at 0 the budget is +// the cheapest bill and the pick is frugal, at 1 it is the dearest and the +// pick is max. +func TestAtKnobAtTheEndsReadsFrugalAndMax(t *testing.T) { + cands := loadCandidates(t) + for _, fam := range []Family{Open, All} { + front := Front(cands, DefaultShapes(), fam) + frugal, _, max := Presets(front) + if got := AtKnob(front, 0); got != frugal { + t.Fatalf("AtKnob at 0 = %+v, want the frugal %+v", got, frugal) + } + if got := AtKnob(front, 1); got != max { + t.Fatalf("AtKnob at 1 = %+v, want the max %+v", got, max) + } + } +} + +// The front is a property of the candidates, not of their order: the running +// for each seat is read in id order, so the same list gives the same front +// however it arrives. +func TestTheSameCandidatesGiveTheSameFrontInAnyOrder(t *testing.T) { + cands := loadCandidates(t) + shapes := DefaultShapes() + one := Front(cands, shapes, All) + + reversed := slices.Clone(cands) + slices.Reverse(reversed) + if two := Front(reversed, shapes, All); !slices.Equal(one, two) { + t.Fatal("the front changed when the candidates arrived reversed") + } +} + +// Two crews that tie in (bill, quality) are one point on the front, and the +// point keeps the crew whose ids read first — worker, then high, then +// mastermind — whichever way round the pool arrived. +func TestTiesInBillAndQualityBreakInIdOrder(t *testing.T) { + // Two candidates identical in every way but the id, and able to sit + // only the mastermind seat: every crew one finishes ties with the crew + // the other finishes. + aaa := model("aaa/x") + aaa.Tools = false + aaa.Images = false + aaa.PromptPrice = 0.1 + aaa.CompletionPrice = 0.2 + aaa.HasCacheRead = false + bbb := aaa + bbb.ID = "bbb/x" + + pool := []Candidate{model("a/m"), model("c/h"), aaa, bbb} + one := Front(pool, DefaultShapes(), All) + if len(one) == 0 { + t.Fatal("the pool must field crews") + } + for _, crew := range one { + if crew.Mastermind != "aaa/x" { + t.Fatalf("a tie went to the wrong twin: %+v", crew) + } + } + + reversed := slices.Clone(pool) + slices.Reverse(reversed) + if two := Front(reversed, DefaultShapes(), All); !slices.Equal(one, two) { + t.Fatal("the tie broke differently when the pool arrived reversed") + } +} + +// The seat's cost is its token mix priced: the prompt price blended with the +// cache-read price by the cache share, and the completion price spread over +// the input-to-output ratio. A model that publishes no cache-read price pays +// the prompt price on the cache share too. +func TestSeatCostPricesTheTokenMix(t *testing.T) { + shape := SeatShape{InOut: 2, CacheShare: 0.5} + published := Candidate{PromptPrice: 2, CompletionPrice: 8, CacheReadPrice: 0.5, HasCacheRead: true} + if got := SeatCost(published, shape); got != 5.25 { + t.Fatalf("SeatCost with a published cache price = %f, want 5.25", got) + } + unpublished := Candidate{PromptPrice: 2, CompletionPrice: 8} + if got := SeatCost(unpublished, shape); got != 6 { + t.Fatalf("SeatCost with no published cache price = %f, want 6", got) + } +} + +// Quality reads each index against the pool's best for that index, so the +// same candidate scores lower in a stronger pool. A row that is not a +// candidate scores nothing, and so does any row against a pool with no +// candidates to raise a scale. +func TestSeatQualityReadsAgainstThePoolsBest(t *testing.T) { + shape := SeatShape{Weights: [3]float64{0.25, 0.25, 0.5}} + pool := []Candidate{ + {ID: "best", Intelligence: 100, Coding: 100, Agentic: 100, PromptPrice: 1}, + {ID: "half", Intelligence: 50, Coding: 100, Agentic: 100, PromptPrice: 1}, + } + if got := SeatQuality(pool[0], shape, pool); got != 100 { + t.Fatalf("the pool's best scored %f, want 100", got) + } + if got := SeatQuality(pool[1], shape, pool); got != 87.5 { + t.Fatalf("the half scored %f, want 87.5", got) + } + + stronger := append(slices.Clone(pool), + Candidate{ID: "top", Intelligence: 200, Coding: 100, Agentic: 100, PromptPrice: 1}) + if got := SeatQuality(pool[1], shape, stronger); got != 81.25 { + t.Fatalf("the half scored %f against a stronger pool, want 81.25", got) + } + + blank := Candidate{ID: "blank", Coding: 0, PromptPrice: 1} + if got := SeatQuality(blank, shape, pool); got != 0 { + t.Fatalf("a row missing an index scored %f, want 0", got) + } + if got := SeatQuality(pool[0], shape, []Candidate{blank}); got != 0 { + t.Fatalf("a pool with no candidates scored %f, want 0", got) + } +} + +// A pool's own rating for a seat moves that seat's quality towards the +// rating's mean, and the more observations back it the farther it moves: a +// rating with a large count across a seat whose catalog quality is 100 and +// whose pool mean is 50 lands somewhere between the two. +func TestALargeRatingMovesASeatTowardsThePoolMean(t *testing.T) { + shape := SeatShape{Weights: [3]float64{1, 0, 0}} + pool := []Candidate{model("m")} + if base := SeatQuality(pool[0], shape, pool); base != 100 { + t.Fatalf("the catalog quality is %f, want 100", base) + } + prior := Prior{Worker: {"m": {Mean: 50, N: 1000}}} + if got := SeatQualityWith(pool[0], shape, pool, Worker, prior); got <= 50 || got >= 100 { + t.Fatalf("a large rating scored %f, want between the pool mean 50 and the catalog 100", got) + } +} + +// The blend carries the rating and the catalog at equal weight at exactly +// PriorWeightAt observations, so the seat reads the midpoint of the two; a +// seat the prior never rates, and a rating with no observations, keep the +// catalog quality. +func TestTheBlendAtTheRatingWeightIsTheMidpoint(t *testing.T) { + shape := SeatShape{Weights: [3]float64{1, 0, 0}} + pool := []Candidate{model("m")} + + half := Prior{Worker: {"m": {Mean: 40, N: PriorWeightAt}}} + if got := SeatQualityWith(pool[0], shape, pool, Worker, half); got != 70 { + t.Fatalf("the blend at N = PriorWeightAt scored %f, want the midpoint 70", got) + } + unrated := Prior{Worker: {"other": {Mean: 40, N: 1000}}} + if got := SeatQualityWith(pool[0], shape, pool, Worker, unrated); got != 100 { + t.Fatalf("a seat the prior never rated scored %f, want the catalog 100", got) + } + empty := Prior{Worker: {"m": {Mean: 40, N: 0}}} + if got := SeatQualityWith(pool[0], shape, pool, Worker, empty); got != 100 { + t.Fatalf("a rating with no observations scored %f, want the catalog 100", got) + } +} + +// A prior is read from role-worded cells: a role that names no seat and a +// count below min_installs are both ignored, and only a role that names a +// seat with enough observations behind it becomes a rating. +func TestPriorFromCellsIgnoresACountBelowMinInstalls(t *testing.T) { + prior := PriorFromCells([]Cell{ + {Role: "worker", Model: "a/kept", Mean: 80, N: 40}, + {Role: "worker", Model: "b/dropped", Mean: 90, N: 10}, + {Role: "engineer", Model: "c/ignored", Mean: 95, N: 100}, + }, 30, nil) + if len(prior) != 1 || len(prior[Worker]) != 1 { + t.Fatalf("prior = %v, want one worker rating", prior) + } + if _, ok := prior[Worker]["a/kept"]; !ok { + t.Fatalf("a rating at the floor was dropped: %v", prior) + } + if _, ok := prior[Worker]["b/dropped"]; ok { + t.Fatalf("a rating below min_installs survived: %v", prior) + } +} + +// The result says which seats a pool's rating entered: a seat whose model the +// prior rated reads measured, and a seat it did not — here the mastermind, +// which no rating names — does not. +func TestMeasuredIsSetOnlyOnTheSeatsThePriorTouched(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h"), model("b/m")} + prior := Prior{ + Worker: {"a/w": {Mean: 100, N: 100}}, + High: {"b/m": {Mean: 100, N: 100}}, + } + front := FrontWith(pool, DefaultShapes(), All, prior) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + wantCrew(t, "frugal", front[0], "a/w|b/m|a/w") + if front[0].Measured != ([3]bool{true, true, false}) { + t.Fatalf("Measured = %v, want the worker and high seats alone", front[0].Measured) + } +} + +// A prior changes quality alone: every bill a crew is given is its seats' +// token mixes priced, exactly as it is without a prior. +func TestAPriorNeverChangesTheBill(t *testing.T) { + pool := []Candidate{model("a/w"), model("c/h"), model("b/m")} + shapes := DefaultShapes() + prior := Prior{ + Worker: {"a/w": {Mean: 20, N: 10_000}}, + High: {"c/h": {Mean: 140, N: 10_000}}, + } + front := FrontWith(pool, shapes, All, prior) + if len(front) == 0 { + t.Fatal("the pool must field crews") + } + for _, crew := range front { + want := 0.0 + for _, seat := range []struct { + model string + seat Seat + }{{crew.Worker, Worker}, {crew.High, High}, {crew.Mastermind, Mastermind}} { + for _, c := range pool { + if c.ID == seat.model { + want += shapes[seat.seat].Volume * SeatCost(c, shapes[seat.seat]) + } + } + } + if math.Abs(crew.Bill-want) > 1e-9 { + t.Fatalf("crew %s|%s|%s bill = %f, want its seats' priced mix %f", + crew.Worker, crew.High, crew.Mastermind, crew.Bill, want) + } + } +} + +// A prior is a property of the candidates, not of their order: with a rating +// present the same pool gives the same front shuffled, rotated and reversed. +func TestAPriorKeepsTheFrontTheSameInAnyOrder(t *testing.T) { + cands := loadCandidates(t) + prior := Prior{Worker: {"z-ai/glm-5.3": {Mean: 90, N: 100}}} + one := FrontWith(cands, DefaultShapes(), All, prior) + if len(one) == 0 { + t.Fatal("the pool must field crews") + } + reversed := slices.Clone(cands) + slices.Reverse(reversed) + if two := FrontWith(reversed, DefaultShapes(), All, prior); !slices.Equal(one, two) { + t.Fatal("the front changed when the candidates arrived reversed") + } + rotated := append(slices.Clone(cands[3:]), cands[:3]...) + if three := FrontWith(rotated, DefaultShapes(), All, prior); !slices.Equal(one, three) { + t.Fatal("the front changed when the pool arrived rotated") + } +} + +// An alias resolved through canonical meets its candidate: a cell spelling a +// model's alias lands on the candidate's canonical id, so the rating reaches +// it, while the same cell with no canonical step lands on the alias and +// touches nothing. +func TestAnAliasResolvedThroughCanonicalMeetsItsCandidate(t *testing.T) { + shape := SeatShape{Weights: [3]float64{1, 0, 0}} + pool := []Candidate{model("z-ai/glm-5.3")} + canonical := func(id string) string { + if id == "glm-5.3" { + return "z-ai/glm-5.3" + } + return id + } + cell := Cell{Role: "worker", Model: "glm-5.3", Mean: 40, N: PriorWeightAt} + resolved := PriorFromCells([]Cell{cell}, 5, canonical) + if got := SeatQualityWith(pool[0], shape, pool, Worker, resolved); got != 70 { + t.Fatalf("the aliased rating scored %f, want the midpoint 70", got) + } + unresolved := PriorFromCells([]Cell{cell}, 5, nil) + if got := SeatQualityWith(pool[0], shape, pool, Worker, unresolved); got != 100 { + t.Fatalf("the unresolved alias scored %f, want the catalog 100 untouched", got) + } +} + +// Two priors fold into one: a rating both hold for a model is weighted by the +// counts behind it, a rating either holds alone is carried whole, and a nil on +// either side is the other side. +func TestMergePriorsFoldsTwoPriorsByObservationCount(t *testing.T) { + if got := MergePriors(nil, nil); got != nil { + t.Fatalf("two nils folded to %v, want nil", got) + } + a := Prior{Worker: {"a/one": {Mean: 80, N: 20}}} + if got := MergePriors(a, nil); got == nil || got[Worker]["a/one"].N != 20 { + t.Fatalf("a nil on one side is not the other: %v", got) + } + b := Prior{High: {"b/two": {Mean: 60, N: 5}}} + if got := MergePriors(nil, b); got == nil || got[High]["b/two"].N != 5 { + t.Fatalf("a nil on one side is not the other: %v", got) + } + + // A model only one prior names is carried whole, in either order. + onlyA := Prior{Worker: {"a/only": {Mean: 70, N: 4}}} + onlyB := Prior{Worker: {"b/only": {Mean: 90, N: 6}}} + got := MergePriors(onlyA, onlyB) + if len(got[Worker]) != 2 { + t.Fatalf("no shared key folded to %v, want the union", got) + } + if r := got[Worker]["a/only"]; r.Mean != 70 || r.N != 4 { + t.Fatalf("a carried rating moved: %v", r) + } + if r := got[Worker]["b/only"]; r.Mean != 90 || r.N != 6 { + t.Fatalf("a carried rating moved: %v", r) + } + + // A model both name: the counts add and the mean is the mean of the means + // weighted by them — (10*50 + 30*90) / 40 = 80. + sharedA := Prior{Worker: {"c/both": {Mean: 50, N: 10}}} + sharedB := Prior{Worker: {"c/both": {Mean: 90, N: 30}}} + if r := MergePriors(sharedA, sharedB)[Worker]["c/both"]; r.Mean != 80 || r.N != 40 { + t.Fatalf("a shared rating folded to %v, want mean 80 over 40", r) + } + + // The priors given are read and never changed. + if r := sharedA[Worker]["c/both"]; r.Mean != 50 || r.N != 10 { + t.Fatalf("the merge moved its first prior: %v", r) + } + if r := sharedB[Worker]["c/both"]; r.Mean != 90 || r.N != 30 { + t.Fatalf("the merge moved its second prior: %v", r) + } +} diff --git a/internal/crewpick/testdata/candidates.json b/internal/crewpick/testdata/candidates.json new file mode 100644 index 0000000000..493a31a519 --- /dev/null +++ b/internal/crewpick/testdata/candidates.json @@ -0,0 +1 @@ +[{"id": "inclusionai/ling-3.0-flash-vl", "open": true, "I": 25, "C": 57, "A": 30, "pi": 0.06, "po": 0.18, "cr": 0.012, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1789056114, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4.1-flash", "open": true, "I": 39.5, "C": null, "A": null, "pi": 0.3, "po": 1.2, "cr": 0.006, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1789021285, "exp": null, "alias": null}, {"id": "openai/gpt-6-astra", "open": false, "I": 52.8, "C": 76.9, "A": 51.5, "pi": 10.0, "po": 50.0, "cr": 1.0, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1788552838, "exp": null, "alias": null}, {"id": "qwen/qwen3.8-max-0902", "open": false, "I": 45.4, "C": 76.2, "A": 56.1, "pi": 2.0, "po": 6.0, "cr": 0.25, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1788469704, "exp": null, "alias": null}, {"id": "google/gemini-3.8-flash", "open": false, "I": 41.2, "C": 76.3, "A": 41.1, "pi": 0.75, "po": 3.75, "cr": 0.075, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1788362056, "exp": null, "alias": null}, {"id": "anthropic/claude-fable-5.1", "open": false, "I": 53.4, "C": 81.6, "A": 58, "pi": 10.0, "po": 50.0, "cr": 0.25, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1788285838, "exp": null, "alias": null}, {"id": "ibm-granite/granite-4.2-8b", "open": true, "I": 11.8, "C": 22.4, "A": 3.7, "pi": 0.06, "po": 0.25, "cr": 0.015, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1788206780, "exp": null, "alias": null}, {"id": "inclusionai/ling-3.0-flash-fin", "open": false, "I": 23, "C": 55.6, "A": 29.3, "pi": 0.06, "po": 0.18, "cr": 0.012, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1787846290, "exp": null, "alias": null}, {"id": "z-ai/glm-5.3-flash", "open": true, "I": 41.9, "C": 71.5, "A": 51.2, "pi": 0.09, "po": 0.3, "cr": 0.018, "ctx": 1310720, "img": true, "tools": true, "so": true, "created": 1787752741, "exp": null, "alias": null}, {"id": "z-ai/glm-5.3", "open": true, "I": 44.9, "C": 74.8, "A": 53.4, "pi": 1.4, "po": 4.4, "cr": 0.26, "ctx": 1310720, "img": false, "tools": true, "so": true, "created": 1787086655, "exp": null, "alias": null}, {"id": "qwen/qwen3.8-27b", "open": true, "I": 33.9, "C": 68.1, "A": 46.5, "pi": 0.21400000000000002, "po": 2.5500000000000003, "cr": 0.15, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1786722910, "exp": null, "alias": null}, {"id": "google/gemini-3.7-flash", "open": false, "I": 39.4, "C": 76.1, "A": 36.4, "pi": 0.75, "po": 3.75, "cr": 0.075, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1786640581, "exp": null, "alias": null}, {"id": "qwen/qwen3.8-2.4t-a95b", "open": true, "I": 40, "C": 71.9, "A": 50.4, "pi": 2.0, "po": 6.0, "cr": 0.25, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1786551702, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4-pro-0813", "open": true, "I": 36.3, "C": 68.8, "A": 42.3, "pi": 1.32, "po": 3.9600000000000004, "cr": 0.044, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1786549364, "exp": null, "alias": null}, {"id": "x-ai/grok-4.6", "open": false, "I": 44.4, "C": 76.8, "A": 53.4, "pi": 2.0, "po": 6.0, "cr": 0.5, "ctx": 500000, "img": true, "tools": true, "so": true, "created": 1786548957, "exp": null, "alias": null}, {"id": "nvidia/nemotron-3.5-lightning", "open": true, "I": 13.6, "C": 26.8, "A": 6.1, "pi": 0.08, "po": 0.19999999999999998, "cr": 0.04, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1786452751, "exp": null, "alias": null}, {"id": "meta/muse-spark-1.2", "open": false, "I": 39.8, "C": 72.2, "A": 44, "pi": 1.25, "po": 4.25, "cr": 0.15, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1785959287, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4-flash-0731", "open": true, "I": 34.5, "C": 69.1, "A": 41.7, "pi": 0.06, "po": 0.12, "cr": 0.012, "ctx": 1310720, "img": false, "tools": true, "so": true, "created": 1785478908, "exp": null, "alias": null}, {"id": "thinkingmachines/inkling-small", "open": true, "I": 26.1, "C": 52.9, "A": 25, "pi": 0.44999999999999996, "po": 1.2, "cr": 0.09999999999999999, "ctx": 1048576, "img": true, "tools": true, "so": false, "created": 1785443117, "exp": null, "alias": null}, {"id": "anthropic/claude-opus-5", "open": false, "I": 50.7, "C": 78, "A": 56.2, "pi": 5.0, "po": 25.0, "cr": 0.5, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1784912544, "exp": null, "alias": null}, {"id": "inclusionai/ling-3.0-flash", "open": true, "I": 20.6, "C": 50.6, "A": 21, "pi": 0.020999999999999998, "po": 0.063, "cr": 0.004200000000000001, "ctx": 262144, "img": false, "tools": true, "so": false, "created": 1784818580, "exp": null, "alias": null}, {"id": "google/gemini-3.6-flash", "open": false, "I": 34.3, "C": 69.2, "A": 30.2, "pi": 0.75, "po": 3.75, "cr": 0.075, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1784646733, "exp": null, "alias": null}, {"id": "google/gemini-3.5-flash-lite", "open": false, "I": 22.7, "C": 49.3, "A": 15.9, "pi": 0.3, "po": 2.5, "cr": 0.03, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1784646726, "exp": null, "alias": null}, {"id": "meituan/longcat-2.0", "open": true, "I": 19.7, "C": 45.3, "A": 15.9, "pi": 0.3, "po": 1.2, "cr": 0.006, "ctx": 1048756, "img": false, "tools": true, "so": false, "created": 1784554658, "exp": null, "alias": null}, {"id": "thinkingmachines/inkling", "open": true, "I": 25.5, "C": 52.1, "A": 24.3, "pi": 1.0, "po": 4.05, "cr": 0.16999999999999998, "ctx": 1048576, "img": true, "tools": true, "so": false, "created": 1784325956, "exp": null, "alias": null}, {"id": "moonshotai/kimi-k3", "open": true, "I": 43.8, "C": 76.2, "A": 50.6, "pi": 3.0, "po": 15.0, "cr": 0.3, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1784215858, "exp": null, "alias": null}, {"id": "meta/muse-spark-1.1", "open": false, "I": 34.3, "C": 71.3, "A": 27.5, "pi": 1.25, "po": 4.25, "cr": 0.15, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1784215741, "exp": null, "alias": null}, {"id": "openai/gpt-5.6-luna", "open": false, "I": 37.5, "C": 71.4, "A": 42.7, "pi": 0.19999999999999998, "po": 1.2, "cr": 0.02, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1783590864, "exp": null, "alias": null}, {"id": "openai/gpt-5.6-terra", "open": false, "I": 42.3, "C": 76.7, "A": 43.7, "pi": 2.0, "po": 12.0, "cr": 0.19999999999999998, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1783590857, "exp": null, "alias": null}, {"id": "openai/gpt-5.6-sol", "open": false, "I": 47.1, "C": 77.4, "A": 50.5, "pi": 2.0, "po": 10.0, "cr": 0.19999999999999998, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1783590850, "exp": null, "alias": null}, {"id": "x-ai/grok-4.5", "open": false, "I": 39.1, "C": 72.4, "A": 42.1, "pi": 2.0, "po": 6.0, "cr": 0.3, "ctx": 500000, "img": true, "tools": true, "so": true, "created": 1783523154, "exp": null, "alias": null}, {"id": "anthropic/claude-sonnet-5", "open": false, "I": 38.4, "C": 71.5, "A": 44.3, "pi": 2.0, "po": 10.0, "cr": 0.19999999999999998, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1782843083, "exp": null, "alias": null}, {"id": "z-ai/glm-5.2", "open": true, "I": 34, "C": 68.8, "A": 39.4, "pi": 1.4, "po": 4.4, "cr": 0.14, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1781631930, "exp": null, "alias": null}, {"id": "moonshotai/kimi-k2.7-code", "open": true, "I": 26.3, "C": 60.8, "A": 22.5, "pi": 0.7062, "po": 3.21, "cr": 0.18, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1781266361, "exp": null, "alias": null}, {"id": "anthropic/claude-fable-5", "open": false, "I": 49.7, "C": 76.5, "A": 51, "pi": 10.0, "po": 50.0, "cr": 1.0, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1781007515, "exp": null, "alias": null}, {"id": "nvidia/nemotron-3-ultra-550b-a55b", "open": true, "I": 23.4, "C": 49.3, "A": 21.7, "pi": 0.625, "po": 3.125, "cr": 0.1875, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1780551208, "exp": null, "alias": null}, {"id": "qwen/qwen3.7-plus", "open": false, "I": 25.8, "C": 55.9, "A": 19.7, "pi": 0.32, "po": 1.28, "cr": 0.064, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1780491783, "exp": null, "alias": null}, {"id": "minimax/minimax-m3", "open": true, "I": 29.6, "C": 58.6, "A": 30.8, "pi": 0.3, "po": 1.2, "cr": 0.06, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1780245374, "exp": null, "alias": null}, {"id": "anthropic/claude-opus-4.8", "open": false, "I": 42, "C": 74.3, "A": 42.6, "pi": 5.0, "po": 25.0, "cr": 0.5, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1779905091, "exp": null, "alias": null}, {"id": "qwen/qwen3.7-max", "open": false, "I": 29.9, "C": 66, "A": 23.9, "pi": 1.475, "po": 4.425, "cr": 0.295, "ctx": 1000000, "img": false, "tools": true, "so": true, "created": 1779376861, "exp": null, "alias": null}, {"id": "google/gemini-3.5-flash", "open": false, "I": 33, "C": 70.1, "A": 27.3, "pi": 1.5, "po": 9.0, "cr": 0.15, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1779193800, "exp": null, "alias": null}, {"id": "x-ai/grok-4.3", "open": false, "I": 25.4, "C": 42.2, "A": 17.2, "pi": 1.25, "po": 2.5, "cr": 0.19999999999999998, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1777591821, "exp": null, "alias": null}, {"id": "mistralai/mistral-medium-3-5", "open": false, "I": 14.9, "C": 46.9, "A": 9.4, "pi": 1.5, "po": 7.5, "cr": null, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1777570439, "exp": null, "alias": null}, {"id": "qwen/qwen3.6-35b-a3b", "open": true, "I": 18.8, "C": 41.9, "A": 15, "pi": 0.09999999999999999, "po": 0.8999999999999999, "cr": 0.049999999999999996, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1777260255, "exp": null, "alias": null}, {"id": "qwen/qwen3.6-27b", "open": true, "I": 21.9, "C": 53.7, "A": 20.1, "pi": 0.3, "po": 2.0, "cr": 0.03, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1777255064, "exp": null, "alias": null}, {"id": "openai/gpt-5.5", "open": false, "I": 38.6, "C": 74.9, "A": 37.3, "pi": 5.0, "po": 30.0, "cr": 0.5, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1777051893, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4-pro", "open": true, "I": 30.9, "C": 59.4, "A": 27.7, "pi": 1.5999999999999999, "po": 3.1999999999999997, "cr": 0.135, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1777000679, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v4-flash", "open": true, "I": 24.8, "C": 52, "A": 27.9, "pi": 0.088606, "po": 0.177212, "cr": 0.017721200000000003, "ctx": 1048576, "img": false, "tools": true, "so": true, "created": 1777000666, "exp": null, "alias": null}, {"id": "tencent/hy3-preview", "open": true, "I": 25.8, "C": 58.8, "A": 25.6, "pi": 0.18, "po": 0.6, "cr": 0.06, "ctx": 262144, "img": false, "tools": true, "so": false, "created": 1776878150, "exp": null, "alias": null}, {"id": "xiaomi/mimo-v2.5-pro", "open": true, "I": 26.4, "C": 60.2, "A": 22.7, "pi": 0.435, "po": 0.87, "cr": 0.0036, "ctx": 1050000, "img": false, "tools": true, "so": true, "created": 1776874273, "exp": null, "alias": null}, {"id": "xiaomi/mimo-v2.5", "open": true, "I": 22.3, "C": 56.8, "A": 17.4, "pi": 0.14, "po": 0.28, "cr": 0.0028, "ctx": 1050000, "img": true, "tools": true, "so": true, "created": 1776874269, "exp": null, "alias": null}, {"id": "moonshotai/kimi-k2.6", "open": true, "I": 27.5, "C": 61.8, "A": 22.1, "pi": 0.95, "po": 4.0, "cr": 0.16, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1776699402, "exp": null, "alias": null}, {"id": "z-ai/glm-5.1", "open": true, "I": 26.4, "C": 55.8, "A": 25.2, "pi": 0.966, "po": 3.036, "cr": 0.1794, "ctx": 204800, "img": false, "tools": true, "so": true, "created": 1775578025, "exp": null, "alias": null}, {"id": "google/gemma-4-31b-it", "open": true, "I": 15.4, "C": 43.4, "A": 6.7, "pi": 0.09, "po": 0.33999999999999997, "cr": 0.049999999999999996, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1775148486, "exp": null, "alias": null}, {"id": "arcee-ai/trinity-large-thinking", "open": true, "I": 10.9, "C": 25.8, "A": 1.2, "pi": 0.25, "po": 0.7999999999999999, "cr": 0.06, "ctx": 262144, "img": false, "tools": true, "so": false, "created": 1775058318, "exp": null, "alias": null}, {"id": "minimax/minimax-m2.7", "open": true, "I": 23.2, "C": 52.6, "A": 16.8, "pi": 0.3, "po": 1.2, "cr": 0.06, "ctx": 204800, "img": false, "tools": true, "so": true, "created": 1773836697, "exp": null, "alias": null}, {"id": "openai/gpt-5.4-nano", "open": false, "I": 21.2, "C": 56.1, "A": 17.7, "pi": 0.19999999999999998, "po": 1.25, "cr": 0.02, "ctx": 400000, "img": true, "tools": true, "so": true, "created": 1773748187, "exp": null, "alias": null}, {"id": "openai/gpt-5.4-mini", "open": false, "I": 24.6, "C": 56.1, "A": 19.7, "pi": 0.75, "po": 4.5, "cr": 0.075, "ctx": 400000, "img": true, "tools": true, "so": true, "created": 1773748178, "exp": null, "alias": null}, {"id": "mistralai/mistral-small-2603", "open": true, "I": 11.5, "C": 26.6, "A": 1.4, "pi": 0.15, "po": 0.6, "cr": 0.015, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1773695685, "exp": null, "alias": null}, {"id": "nvidia/nemotron-3-super-120b-a12b", "open": true, "I": 13.6, "C": 37.7, "A": 4.1, "pi": 0.08, "po": 0.44999999999999996, "cr": null, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1773245239, "exp": null, "alias": null}, {"id": "inception/mercury-2", "open": false, "I": 11.5, "C": 31.1, "A": 4, "pi": 0.25, "po": 0.75, "cr": 0.024999999999999998, "ctx": 128000, "img": false, "tools": true, "so": true, "created": 1772636275, "exp": null, "alias": null}, {"id": "google/gemini-3.1-flash-lite-preview", "open": false, "I": 16, "C": 34.7, "A": 3.2, "pi": 0.25, "po": 1.5, "cr": 0.024999999999999998, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1772512673, "exp": null, "alias": null}, {"id": "qwen/qwen3.5-122b-a10b", "open": true, "I": 16.2, "C": 45.7, "A": 9.6, "pi": 0.26, "po": 2.08, "cr": null, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1772053789, "exp": null, "alias": null}, {"id": "google/gemini-3.1-pro-preview", "open": false, "I": 30.4, "C": 68.8, "A": 10.3, "pi": 2.0, "po": 12.0, "cr": 0.19999999999999998, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1771509627, "exp": null, "alias": null}, {"id": "anthropic/claude-sonnet-4.6", "open": false, "I": 30.5, "C": 63, "A": 33.1, "pi": 3.0, "po": 15.0, "cr": 0.3, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1771342990, "exp": null, "alias": null}, {"id": "qwen/qwen3.5-397b-a17b", "open": true, "I": 19.1, "C": 48.2, "A": 10.6, "pi": 0.55, "po": 3.5, "cr": 0.22499999999999998, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1771223018, "exp": null, "alias": null}, {"id": "qwen/qwen3-coder-next", "open": true, "I": 10.1, "C": 36.2, "A": 3.6, "pi": 0.12, "po": 0.7999999999999999, "cr": 0.07, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1770164101, "exp": null, "alias": null}, {"id": "upstage/solar-pro-3", "open": false, "I": 7.8, "C": 16.2, "A": 1.4, "pi": 0.15, "po": 0.6, "cr": 0.015, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1769481200, "exp": null, "alias": null}, {"id": "nvidia/nemotron-3-nano-30b-a3b", "open": true, "I": 8.9, "C": 14.4, "A": 1, "pi": 0.06, "po": 0.24, "cr": null, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1765731275, "exp": null, "alias": null}, {"id": "mistralai/devstral-2512", "open": true, "I": 9.4, "C": 31.3, "A": 4.9, "pi": 0.39999999999999997, "po": 2.0, "cr": 0.04, "ctx": 262144, "img": false, "tools": true, "so": true, "created": 1765285419, "exp": null, "alias": null}, {"id": "mistralai/ministral-14b-2512", "open": true, "I": 6, "C": 14.4, "A": 1.1, "pi": 0.19999999999999998, "po": 0.19999999999999998, "cr": 0.02, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1764681735, "exp": null, "alias": null}, {"id": "mistralai/ministral-8b-2512", "open": true, "I": 5.5, "C": 9.7, "A": 0.6, "pi": 0.15, "po": 0.15, "cr": 0.015, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1764681654, "exp": null, "alias": null}, {"id": "mistralai/ministral-3b-2512", "open": true, "I": 4.8, "C": 4.8, "A": 0.8, "pi": 0.09999999999999999, "po": 0.09999999999999999, "cr": 0.01, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1764681560, "exp": null, "alias": null}, {"id": "mistralai/mistral-large-2512", "open": false, "I": 9.7, "C": 20.1, "A": 2.4, "pi": 0.5, "po": 1.5, "cr": 0.049999999999999996, "ctx": 262144, "img": true, "tools": true, "so": true, "created": 1764624472, "exp": null, "alias": null}, {"id": "anthropic/claude-haiku-4.5", "open": false, "I": 17.6, "C": 43.9, "A": 10.3, "pi": 1.0, "po": 5.0, "cr": 0.09999999999999999, "ctx": 200000, "img": true, "tools": true, "so": true, "created": 1760547638, "exp": null, "alias": null}, {"id": "anthropic/claude-sonnet-4.5", "open": false, "I": 21.2, "C": 52.1, "A": 17.5, "pi": 3.0, "po": 15.0, "cr": 0.3, "ctx": 1000000, "img": true, "tools": true, "so": true, "created": 1759161676, "exp": null, "alias": null}, {"id": "deepseek/deepseek-v3.1-terminus", "open": true, "I": 15.4, "C": 43.5, "A": 8.9, "pi": 0.27, "po": 1.0, "cr": 0.135, "ctx": 163840, "img": false, "tools": true, "so": true, "created": 1758548275, "exp": null, "alias": null}, {"id": "qwen/qwen3-30b-a3b-thinking-2507", "open": true, "I": 9.8, "C": 12.1, "A": 0.9, "pi": 0.19999999999999998, "po": 2.4, "cr": null, "ctx": 81920, "img": false, "tools": true, "so": false, "created": 1756399192, "exp": null, "alias": null}, {"id": "mistralai/mistral-medium-3.1", "open": false, "I": 9.5, "C": 20.5, "A": 3.1, "pi": 0.39999999999999997, "po": 2.0, "cr": 0.04, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1755095639, "exp": null, "alias": null}, {"id": "openai/gpt-5-mini", "open": false, "I": 17.4, "C": 15.6, "A": 8.9, "pi": 0.25, "po": 2.0, "cr": 0.024999999999999998, "ctx": 400000, "img": true, "tools": true, "so": true, "created": 1754587407, "exp": null, "alias": null}, {"id": "openai/gpt-oss-120b", "open": true, "I": 12.3, "C": 30.4, "A": 6.2, "pi": 0.037, "po": 0.16999999999999998, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1754414231, "exp": null, "alias": null}, {"id": "openai/gpt-oss-20b", "open": true, "I": 9, "C": 20.7, "A": 1.4, "pi": 0.03, "po": 0.13, "cr": 0.03, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1754414229, "exp": null, "alias": null}, {"id": "qwen/qwen3-235b-a22b-thinking-2507", "open": true, "I": 12.7, "C": 22.1, "A": 1.3, "pi": 0.22999999999999998, "po": 2.3, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": false, "created": 1753449557, "exp": null, "alias": null}, {"id": "google/gemini-2.5-pro", "open": false, "I": 16.7, "C": 33.3, "A": 3.5, "pi": 1.25, "po": 10.0, "cr": 0.125, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1750169544, "exp": null, "alias": null}, {"id": "qwen/qwen3-8b", "open": true, "I": 5.2, "C": 9, "A": 0.8, "pi": 0.117, "po": 0.45499999999999996, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": false, "created": 1745876632, "exp": null, "alias": null}, {"id": "qwen/qwen3-14b", "open": true, "I": 6.4, "C": 13.8, "A": 0.9, "pi": 0.12, "po": 0.24, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1745876478, "exp": null, "alias": null}, {"id": "qwen/qwen3-32b", "open": true, "I": 7.2, "C": 15.3, "A": 0.9, "pi": 0.08, "po": 0.28, "cr": null, "ctx": 131072, "img": false, "tools": true, "so": true, "created": 1745875945, "exp": null, "alias": null}, {"id": "meta-llama/llama-4-maverick", "open": true, "I": 9.3, "C": 16.3, "A": 0.6, "pi": 0.1875, "po": 0.6525, "cr": null, "ctx": 1048576, "img": true, "tools": true, "so": true, "created": 1743881822, "exp": null, "alias": null}, {"id": "meta-llama/llama-4-scout", "open": true, "I": 6.5, "C": 8.2, "A": 0.5, "pi": 0.09999999999999999, "po": 0.3, "cr": null, "ctx": 1310720, "img": true, "tools": true, "so": true, "created": 1743881519, "exp": null, "alias": null}, {"id": "deepseek/deepseek-chat-v3-0324", "open": true, "I": 9.7, "C": 21.2, "A": 0.8, "pi": 0.25, "po": 1.0, "cr": null, "ctx": 163840, "img": false, "tools": true, "so": true, "created": 1742824755, "exp": null, "alias": null}, {"id": "google/gemma-3-12b-it", "open": true, "I": 3.8, "C": 5.8, "A": 0.1, "pi": 0.049999999999999996, "po": 0.15, "cr": null, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1741902625, "exp": null, "alias": null}, {"id": "cohere/command-a", "open": true, "I": 13.9, "C": 27.8, "A": 3.6, "pi": 2.5, "po": 10.0, "cr": null, "ctx": 256000, "img": false, "tools": false, "so": true, "created": 1741894342, "exp": null, "alias": null}, {"id": "google/gemma-3-27b-it", "open": true, "I": 4.9, "C": 10.1, "A": 0.1, "pi": 0.08, "po": 0.44999999999999996, "cr": 0.04, "ctx": 131072, "img": true, "tools": true, "so": true, "created": 1741756359, "exp": null, "alias": null}, {"id": "openai/o3-mini-high", "open": false, "I": 11, "C": 16.3, "A": 0.9, "pi": 1.1, "po": 4.4, "cr": 0.55, "ctx": 200000, "img": false, "tools": true, "so": true, "created": 1739372611, "exp": null, "alias": null}, {"id": "deepseek/deepseek-r1", "open": true, "I": 11.4, "C": 24.6, "A": 1.1, "pi": 0.7, "po": 2.5, "cr": null, "ctx": 64000, "img": false, "tools": true, "so": false, "created": 1737381095, "exp": null, "alias": null}] \ No newline at end of file diff --git a/internal/manual/chat/commands.md b/internal/manual/chat/commands.md index 9b262d4756..b3ee4bd857 100644 --- a/internal/manual/chat/commands.md +++ b/internal/manual/chat/commands.md @@ -673,7 +673,7 @@ The `crew` line sits directly under `model` and reads the preset word — or `cu the three classes: ``` -crew max · brain kimi-k3 · hands glm-5.3 · checks kimi-k3 +crew max · brain claude-opus-5 · hands glm-5.3 · checks claude-fable-5.1 ``` The crew is **not on the status line**. It was one short segment there — `crew max`, or @@ -1428,12 +1428,12 @@ opens the six-seat reading, bottom-anchored like the model picker. From the top: the five models codeaf uses on its own behalf — not the one you chat with you talk to · deepseek-v4-flash family ‹ open models · all models › - frugal — deepseek works, glm-5.3 thinks · pennies a day - reflex mistralai/mistral-nemo · small work deepseek/deepseek-v4-flash-0731 · worker deepseek/deepseek-v4-flash-0731 · careful work z-ai/glm-5.3-flash · mastermind z-ai/glm-5.3 -› balanced — glm-flash works, kimi-k3 checks and thinks - reflex mistralai/mistral-nemo · small work deepseek/deepseek-v4-flash-0731 · worker z-ai/glm-5.3-flash · careful work moonshotai/kimi-k3 · mastermind moonshotai/kimi-k3 - max — glm-5.3 works, kimi-k3 thinks and checks - reflex mistralai/mistral-nemo · small work deepseek/deepseek-v4-flash-0731 · worker z-ai/glm-5.3 · careful work moonshotai/kimi-k3 · mastermind moonshotai/kimi-k3 + frugal — glm-flash works and thinks, qwen-max checks + reflex google/gemini-2.5-flash · small work deepseek/deepseek-v4-flash-0731 · worker z-ai/glm-5.3-flash · careful work qwen/qwen3.8-max-0902 · mastermind z-ai/glm-5.3-flash +› balanced — glm-flash works, fable checks, opus thinks + reflex google/gemini-2.5-flash · small work deepseek/deepseek-v4-flash-0731 · worker z-ai/glm-5.3-flash · careful work anthropic/claude-fable-5.1 · mastermind anthropic/claude-opus-5 + max — glm-5.3 works, fable checks, opus thinks + reflex google/gemini-2.5-flash · small work deepseek/deepseek-v4-flash-0731 · worker z-ai/glm-5.3 · careful work anthropic/claude-fable-5.1 · mastermind anthropic/claude-opus-5 each of the five can be pinned on its own in /settings → Providers ``` @@ -1460,7 +1460,7 @@ page. `/crew frugal`, `/crew balanced` or `/crew max` sets the five and confirms in one line: ``` -crew → max · brain kimi-k3 · hands glm-5.3 · checks kimi-k3 · you are still talking to deepseek-v4-flash — /model changes that +crew → max · brain claude-opus-5 · hands glm-5.3 · checks claude-fable-5.1 · you are still talking to deepseek-v4-flash — /model changes that ``` **The model ids are drawn brighter than the words around them.** `crew →`, the preset diff --git a/internal/manual/chat/models-and-cost.md b/internal/manual/chat/models-and-cost.md index e51f418e42..26c6a26eb6 100644 --- a/internal/manual/chat/models-and-cost.md +++ b/internal/manual/chat/models-and-cost.md @@ -331,60 +331,67 @@ preset: | class | as shipped | | --- | --- | -| reflex | `mistralai/mistral-nemo` | +| reflex | `google/gemini-2.5-flash` | | small work | `deepseek/deepseek-v4-flash-0731` | | worker | `z-ai/glm-5.3-flash` | -| careful work | `moonshotai/kimi-k3` | -| mastermind | `moonshotai/kimi-k3` | - -They are all open-weight models, and none of them is the model you are talking to. A crew -that followed your conversation would put the most expensive model in the build on the -cheapest questions in it — a call made twice every turn on a frontier model is a bill nobody -agreed to. Closed models that are cheaper on their own vendor's platform than through the -router are not in any open-family preset; the `all` family is where they are, and you -can still pin one on any row. -You can also connect that vendor yourself as a direct service; the +| careful work | `anthropic/claude-fable-5.1` | +| mastermind | `anthropic/claude-opus-5` | + +None of them is the model you are talking to. A crew that followed your conversation would +put the most expensive model in the build on the cheapest questions in it — a call made +twice every turn on a frontier model is a bill nobody agreed to. That is also why the two +seats that read every turn stay on near-free models while the frontier ids sit on the seats +that answer a handful of times. You can pin any vendor's model on any row yourself, and you +can connect that vendor as a direct service; the [services page](services.md) explains its names, limits and missing Phase 1 cost record. -**The `model family` row** (`models.crew.source`, just under the crew row) decides which family -the three preset words draw from. `open` is the default and is the table above: open weights only, -so a crew nobody chose is never a bet on one vendor's pricing. `all` reads the same three words, -`frugal`, `balanced` and `max`, off the whole catalog instead, closed and frontier models included, -and costs what those models cost. Flip the row and the seats nobody pinned move with it at once, +**The `model family` row** (`models.crew.source`) decides which family +the three preset words draw from. `all` is the default and is the table above: the whole +catalog, closed and frontier models included, costing what those models cost. `open` reads the +same three words, `frugal`, `balanced` and `max`, off the open-weight rows only, so no seat is a +bet on one vendor's pricing. Flip the row and the seats nobody pinned move with it at once, because an unwritten seat is the default crew resolved in the family you are on; the rows already on disk keep their ids until you pick the crew again, and the crew word reads `custom` while they match no preset in the family you flipped to. -Under `all`, the same three words resolve to these: +Under `open`, the same three words resolve to these: | class | frugal | balanced | max | | --- | --- | --- | --- | -| reflex | `google/gemini-2.5-flash` | `google/gemini-2.5-flash` | `google/gemini-2.5-flash` | +| reflex | `mistralai/mistral-nemo` | `mistralai/mistral-nemo` | `mistralai/mistral-nemo` | | small work | `deepseek/deepseek-v4-flash-0731` | `deepseek/deepseek-v4-flash-0731` | `deepseek/deepseek-v4-flash-0731` | -| worker | `openai/gpt-5.6-sol` | `openai/gpt-5.6-sol` | `anthropic/claude-fable-5.1` | -| careful work | `google/gemini-3.8-flash` | `anthropic/claude-opus-5` | `openai/gpt-6-astra` | -| mastermind | `anthropic/claude-opus-5` | `anthropic/claude-fable-5.1` | `anthropic/claude-fable-5.1` | - -Two of the columns spend differently here. The worker is the same model in `frugal` and -`balanced`, and only `max` buys the frontier coder, because on the catalog the coding ceiling is -the one expensive step. What `frugal` to `balanced` buys instead is the careful seat and a -smarter mastermind. The `max` mastermind sits on the worker's own model on purpose: it is the -catalog's ceiling, so there is no bigger planner to buy above it. - -**Why the open ids.** They were picked off the catalog's own published scores (OpenRouter -republishes Artificial Analysis's coding and agentic indexes on every model row) against -blended price, open weights only, and the `frugal` mastermind and the `balanced` careful and mastermind seats were -picked on 2026-09-16, the rest on 2026-09-01. The worker seat is the dial: `glm-5.3-flash` -scores 58 on the agentic index and 72 on coding at about $0.12 per million tokens blended, -one point under `glm-5.3` at a twentieth of its price, and it can see images. The -mastermind is the one seat whose price may be the model rather than the rung: it answers a -few times a task, so `frugal` pays for `glm-5.3` while the work below it stays on -`glm-5.3-flash` and DeepSeek. -The careful class is always a different vendor from the worker and always sees images: -`moonshotai/kimi-k3` scores 76 on coding and is the open field's strongest checker, so -`balanced` and `max` put it on both the careful and the mastermind seat. The small-work row is pinned to the July build of DeepSeek V4 Flash on purpose: -the bare `deepseek/deepseek-v4-flash` id resolves to the April build, and the July build at -the same price scores thirteen coding points higher. +| worker | `z-ai/glm-5.3-flash` | `z-ai/glm-5.3-flash` | `z-ai/glm-5.3` | +| careful work | `z-ai/glm-5.3-flash` | `moonshotai/kimi-k3` | `moonshotai/kimi-k3` | +| mastermind | `z-ai/glm-5.3-flash` | `z-ai/glm-5.3` | `z-ai/glm-5.3` | + +The worker column is the dial in that family — it holds `glm-5.3-flash` through `balanced`, and +`max` moves it to `glm-5.3` — because it is the seat that pays most of a task's bill and a +preset that moved every other seat would change everything about a task except its cost. + +**Why these ids.** Every row of the catalog was placed on two axes on 2026-09-16: the bill a +seat's own call shape runs up, built from the catalog's published prompt, completion and +cache-read prices, against that seat's quality, taken from its published intelligence, coding +and agentic indexes. The call shape is part of the price, so the seats were priced apart — the +worker and the careful seats as long cached loops, where a large prompt is read back turn +after turn and the cache-read price carries most of the weight, and the mastermind as one-shot +calls, where the prompt is paid in full each time and there are few of them. Each preset then +takes, for each seat, a point on the pareto front of that plot at the bill it is willing to +run: nothing on the front costs less at the same quality, and nothing at the same bill scores +higher. The two families are the same plot over two sets of rows, which is why the columns do +not climb together in either of them. + +In the `all` family the worker stays on `glm-5.3-flash` through `balanced`, because the worker +seat carries most of a task's tokens: a step there multiplies through the whole bill, where a +step on the careful or the mastermind seat is paid a handful of times. So `frugal` to +`balanced` spends on those two low-volume seats, and `max` moves the worker itself. The +careful and mastermind seats are settled by `balanced` and stay there through `max`. + +The careful class always sees images — the vision role rides that row — and is a different +vendor from the worker in every preset of the `all` family; the open family's `frugal` row +is the one standing exception, with worker and careful both on `glm-5.3-flash`, because at +that bill the open-weight front has no second vendor for the careful seat. The small-work row is pinned to +the July build of DeepSeek V4 Flash on purpose: the bare `deepseek/deepseek-v4-flash` id +resolves to the April build, and the July build costs the same. **Clearing a row is still an answer.** A class you empty on purpose reads `follows the conversation`, and every role on it runs on the model you are talking to. That @@ -393,20 +400,22 @@ say rather than the default. ### The three presets +These are the three in the family you are on unless you changed the `model family` row — the +default one, `all`: + | | frugal | balanced | max | | --- | --- | --- | --- | -| reflex | `mistral-nemo` | `mistral-nemo` | `mistral-nemo` | +| reflex | `gemini-2.5-flash` | `gemini-2.5-flash` | `gemini-2.5-flash` | | small work | `deepseek-v4-flash-0731` | `deepseek-v4-flash-0731` | `deepseek-v4-flash-0731` | -| worker | `deepseek-v4-flash-0731` | `glm-5.3-flash` | `glm-5.3` | -| careful work | `glm-5.3-flash` | `kimi-k3` | `kimi-k3` | -| mastermind | `glm-5.3` | `kimi-k3` | `kimi-k3` | +| worker | `glm-5.3-flash` | `glm-5.3-flash` | `glm-5.3` | +| careful work | `qwen3.8-max-0902` | `claude-fable-5.1` | `claude-fable-5.1` | +| mastermind | `glm-5.3-flash` | `claude-opus-5` | `claude-opus-5` | -- **frugal**: deepseek works, glm-5.3 thinks · pennies a day -- **balanced**: glm-flash works, kimi-k3 checks and thinks -- **max**: glm-5.3 works, kimi-k3 thinks and checks +- **frugal**: glm-flash works and thinks, qwen-max checks +- **balanced**: glm-flash works, fable checks, opus thinks +- **max**: glm-5.3 works, fable checks, opus thinks -The worker column climbs the open-weight front one step per preset, because it is the seat -that pays most of a task's bill. The reflex and small-work columns never vary — they are the +The reflex and small-work columns never vary — they are the same near-free models in all three — so `/crew` never names them: the confirmation and `/status` say **brain**, **hands** and **checks**, which are the mastermind, the worker and the careful class. @@ -418,9 +427,13 @@ enter applies and esc cancels. If the five classes make a custom crew, no row is the chooser says picking one puts all five back. `/crew max` still sets it directly and confirms in one line, which ends `· you are still talking to deepseek-v4-flash — /model changes that` — naming the conversation's own model by id, because the crew changes -nothing about it and the model segment on the status line goes on saying what it said before. -The **crew** row in `/settings` → Providers is the same thing: enter or space walks it -frugal → balanced → max. +nothing about it and the model segment on the status line goes on saying what it said +before. `/crew learn`, `/crew catalog` and `/crew table` move the **picked from** row and +nothing else — the three words the row takes, answered without touching a model id — and +when the pick is off `table` the crew word says so beside the preset: `balanced · learn` +in `/status`, `crew balanced · learn` on the status line. The **crew** row in `/settings` +→ Providers is the same thing: enter or space walks it frugal → balanced → max, and the +**picked from** row under it walks table → catalog → learn. **The crew row is not stored — it is worked out from the five.** Answer any one of the five rows yourself and the crew row reads `custom`, because that is what is true. `/crew balanced` @@ -428,6 +441,61 @@ puts all five back in one write. A profile that applied a crew before the worker existed reads `custom` until a preset is applied again, because its four old rows and the new fifth are not any of the three. +### Where the seats are picked from — table, catalog, learn + +The **picked from** row (`models.crew.pick`, just under the crew row) says where the +seats' models come from when a class row does not hold a model id of its own. The crew +row above it still says how much to spend; this row says where the models for that +money are read from: + +- **table** — the rows we measured: the ids this build shipped with, the same ones every + preset table holds. This is the default, and it is what an unwritten class has always + read. +- **catalog** — recomputed from today's published prices and scores at your crew's + budget, on every read. Nothing is stored; a catalog that moves moves the seat with it, + and your profile never holds a model id this build chose for you. +- **learn** — the catalog computation plus the Model Pool's measurements and your own + judged runs, carried as a quality rating the better-measured models read on top of + their published scores. + +`catalog` and `learn` are Pareto crewing: the crew is picked on the cost-quality front, +per role and per task, from evidence rather than from a fixed table. + +A pick moves the three seats the presets dial — **worker**, **careful work** and +**mastermind** — and never the two that read every turn: **reflex** and **small work** +keep their near-free ids, the same ones in every preset. + +**A model you typed by hand wins.** The pick answers for the seats nobody named. A class +row holding a model id that is not the preset's own keeps it, spelled as you typed it; +a row holding the preset's own id is the preset answering rather than a pin, and the +pick computes it. **A flag or an environment variable still outranks the pick** — the +pick reads the profile, and `--model` and `CODEAF_MODEL` are what an invocation said. + +When the catalog cannot compute a seat — no catalog yet, or no pick off its front — the +seat falls back to the table row for your preset, never to `auto` and never to empty. A +seat the pick computed names it where the preset would be: `crew balanced, computed +from the catalog` under **catalog**, `crew balanced, learned` under **learn**. + +**The per-seat alias is the bare word `auto`.** Any of the five class rows may hold the +bare word `auto` instead of a model id, case folded. The seat's model is then +**computed from the catalog** — the three published capability indexes against the three +published prices, under that seat's own call shape — every time the row is read. The +word stays on disk; the id is worked out on every read. + +**The budget it computes at comes from the other four rows.** `auto` has no opinion about +cost of its own, so it runs at whatever preset the rows around it name: four rows that are +`balanced`'s make an `auto` row a balanced seat, four that are `max`'s make it a max seat. + +**When the other rows match more than one preset, `balanced` wins.** It is the default +preset, and the budget an undecided profile runs at is the budget an `auto` seat runs at. +This is not a rare corner: `max` differs from `balanced` only in the worker seat, so a +crew with `auto` on the worker and the other four rows as shipped matches both, and reads +as `balanced`. Pin the worker to `max`'s own id and put the `auto` row on a seat above it +if you want a computed seat at `max`'s budget. + +A seat on a computed row names both facts where a seat is shown — `crew balanced, +computed from the catalog` — so the reading is never a guess. + ### The roles under each class Directly under the **pinned roles** row the panel lists **every registered role**, grouped @@ -510,7 +578,7 @@ running keeps the model it was admitted on. **Where to read the crew back:** - `/status` prints a `crew` line directly under `model`: - `crew max · brain kimi-k3 · hands glm-5.3 · checks kimi-k3`. The word is + `crew max · brain claude-opus-5 · hands glm-5.3 · checks claude-fable-5.1`. The word is the preset, or `custom` when the five classes are your own arrangement. **brain** is the mastermind, **hands** is the worker, **checks** is the careful class. - `/settings` → Providers has the **crew** row above the five class rows. @@ -574,7 +642,7 @@ was silently lost the moment the same brain ran from a script. Each of those runs opens by saying which voice answered, so nothing has to be guessed at: ``` -models: work deepseek/deepseek-v4-flash-0731 (crew frugal) · plan z-ai/glm-5.3-flash (crew frugal) +models: work z-ai/glm-5.3-flash (crew frugal) · plan z-ai/glm-5.3-flash (crew frugal) ``` Two details worth knowing. A crew answers only once you have actually set one — a profile @@ -678,7 +746,7 @@ No. `/crew max` moves the five crew seats and leaves the model you talk to exact was. The confirmation names it: ``` -crew → max · brain kimi-k3 · hands glm-5.3 · checks kimi-k3 · you are still talking to deepseek-v4-flash — /model changes that +crew → max · brain claude-opus-5 · hands glm-5.3 · checks claude-fable-5.1 · you are still talking to deepseek-v4-flash — /model changes that ``` `/model`, `/model ` or the model row in `/settings` are the only ways to change the @@ -2246,7 +2314,7 @@ The `crew` line sits directly under `model` and reads the preset word — or `cu the three classes after it: ``` -crew max · brain kimi-k3 · hands glm-5.3 · checks kimi-k3 +crew max · brain claude-opus-5 · hands glm-5.3 · checks claude-fable-5.1 ``` On the live status line the crew is one short segment — `crew max`, or `crew custom` — at @@ -3499,8 +3567,8 @@ on the `prompt profile` row or pinned it for the launch. Nothing about a model's licence, its vendor, its name or which crew seat it sits in makes a session lean. -So an open-weight model with a large window is NOT lean. `deepseek-v4-flash` and -`glm-5.3-flash` are served with 128,000 tokens of room, so they get the full +So an open-weight model with a large window is NOT lean. `glm-5.3-flash` and +`glm-5.3` are served with 128,000 tokens of room, so they get the full page, the full tool list and saved memories, exactly like any other 128,000-token model — including when they are the model your crew preset picked for the `worker` seat, and including when you then choose that same model in diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index ce109bbdba..d4803d0b85 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -213,7 +213,8 @@ stdout, always parseable, printed even when the run failed**: "steps": 3, "run": "0123456789abcdef", "calls": 47, - "rounds": 2 + "rounds": 2, + "redispatches": 1 } ``` @@ -227,11 +228,13 @@ stdout, always parseable, printed even when the run failed**: | `spend_usd` | what it cost, whole, in dollars | | `tokens` | `{"in": …, "out": …}` | | `seconds` | wall clock | +| `core_done_seconds` | when the requested work was first found done, in seconds from the start; absent when the gate never said so | | `model` | the model the work ran on | | `steps` | how many pieces of work ran — `do`'s nodes, `exec`'s turns. A saved program does not measure it: the key is still there, holding `0`, and that `0` is a measurement nobody took rather than a count of none | | `run` | this invocation's id. It names the folder `--debug` writes into, and every row this run wrote into `~/.codeaf/logs/calls.jsonl` carries it too — so `codeaf logs --run ` is how you get from this object to the calls behind it. Empty on a verb that opened no run of its own | | `calls` | how many model calls the run made, counted whether or not the call log is switched on. It is the figure you would otherwise count by hand in `calls.jsonl` | | `rounds` | how many times the run went back for **more work** after looking at what it had. One is the ordinary shape; eight is a run that kept finding more to do, and it is the number that explains a bill nothing else here accounts for. `exec` does not plan and a saved program does not grow, so both hold `0` — a measurement nobody took, the way `steps` does | +| `redispatches` | how many times one of the run's nodes was sent round again **in place** after running out of the room it was granted — back on the queue to carry on from what it had already banked, rather than grown around. Present only when it happened: a run that never re-dispatched a node carries no key at all, and neither does a verb with no plan nodes | **Within a release a field is never removed and never changes meaning; new fields may appear.** `stop` is the field to read for *why*; the exit code only says how much is wrong. @@ -551,6 +554,72 @@ machine would run work on. **On a machine with no evidence the whole answer is o No competence evidence yet. ``` +## The Model Pool — what this machine reads from it, with codeaf pool + +codeaf picks its models against the public Model Pool: a signed index of +measured models that your runs improve. Nothing about your code ever leaves +the machine — what is shared is a measurement of the run, not the work. One +setting answers for all of it, `model_pool` in `/settings`, with three +values: `on` reads and sends, `read` uses the pool and sends nothing, `off` +does neither. It defaults to `on`. `CODEAF_MODEL_POOL` pins the same word +from the shell, and on a CI machine with neither set codeaf reads but does +not send. + +``` +codeaf pool [show|status|verify] [--json] [--key key] +``` + +`show` — also what bare `codeaf pool` prints — is the reading form: the mode +and the addresses in force with the word saying where each came from +(`default`, `setting`, `env` or `ci`), then what index is cached, how old +it is and how many cells it holds, or `no index cached yet · built-in +seed of `. The binary carries a seed index of our own scored runs, +read until a fresher signed one is cached. Your install also keeps +the scores its judge gave in `own.json` +under the pool directory — `show` and `status` say what that sheet holds — and +the crew reads them beside the index. `status` adds what is waiting to be sent +and whether the mode allows sending and reading. `codeaf pool status` also +says whether the relay answered, and whether the mirror did, and what the +last judge did — which model, which seats it scored, or why it failed. `--json` prints +the same answer as one object; `show` reads nothing off the network. + +**The scores start here.** In a conversation, after a task lands, a model +outside the crew is asked to score each seat the work ran on — the worker that +carried it, and the seat that checked it when there was one. The scores stay +in your install's own sheet (`own.json`) and feed the very next crew pick; +nothing else reads them. With `model_pool` set to `on` the same scores also +wait in `outbox.jsonl` beside the sheet, to leave with the pool's other +measurements; `read` keeps them local, and `off` asks no judge at all and +writes nothing. The call itself is billed to the `judge` seat, so it shows up +in the spend pages beside the crew seats rather than inside a task's own +cost. + +With `model_pool` on, the rows leave for the relay after each judged run and +once more at start-up, under this install's own nonce and nothing else. The +index is fetched once a day, checked against the key built into the binary — +or the key in `models.pool.public_key` when one is set — and a changed +document is read at the next start. + +`verify` fetches a fresh index and checks its detached ed25519 signature, +then prints the version whose signature checked out: + +``` +signature good: version 7, generated 2026-09-10, 3 metrics +``` + +It wants a public key: `--key `, repeatable, or +one built into the build. The build carries the index signer's key, so +`verify` works as it stands; `--key` checks under that key alone — the +build's key is not consulted beside it, so a document signed under any +other key does not verify. A fetch or a signature that fails is exit 1; a +`verify` on a machine whose setting is `off`, or whose +`models.pool.public_key` does not decode, is refused with exit 2 and +fetches nothing. + +A private relay is a copy of `relay/` deployed to your own account with your +own keypair, and `CODEAF_MODEL_POOL_RELAY_URL` with `models.pool.public_key` +(or `CODEAF_MODEL_POOL_PUBLIC_KEY`) point an install at it. + ## What is still running in the background — codeaf services, and stopping one `codeaf services` lists the long-running processes started here and not yet stopped — a dev diff --git a/internal/manual/chat/what-i-remember.md b/internal/manual/chat/what-i-remember.md index c5775d05d5..e35c945df7 100644 --- a/internal/manual/chat/what-i-remember.md +++ b/internal/manual/chat/what-i-remember.md @@ -539,7 +539,8 @@ here. The two that ride every message are both on the cheapest of the five crew classes — the `reflex` class, which exists precisely because a call made twice a turn is a different economy from one made once a session. It ships pointed at -`mistralai/mistral-nemo`. Each goes out with a short +`google/gemini-2.5-flash`, and at `mistralai/mistral-nemo` under the `open` +crew family. Each goes out with a short prompt and a 200-token ceiling, and each is asked to answer in a few words of JSON. diff --git a/internal/manual/pages/models.md b/internal/manual/pages/models.md index b757745a76..a186a093e8 100644 --- a/internal/manual/pages/models.md +++ b/internal/manual/pages/models.md @@ -90,11 +90,15 @@ that works, one that plans — and both resolve the same way. First answer wins: The crew is the same one `/crew` sets in the chat, so a machine told `frugal` there runs frugal here. It answers only where a crew has actually been set; an untouched profile takes the build's default. `CODEAF_HOME` and `CODEAF_PROFILE_DIR` decide which profile is asked. +A seat computed from the catalog says so where the preset would be — `crew frugal, +computed from the catalog` — and one computed with what your runs measured says `crew +frugal, learned`; the **picked from** row in the chat's settings decides which, and a +pick of `catalog` or `learn` seats a run from the shell exactly as it seats the chat. Every one of those runs opens with a line on stderr naming both seats and what chose each: ``` -models: work deepseek/deepseek-v4-flash (crew frugal) · plan qwen/qwen3.8-27b (crew frugal) +models: work z-ai/glm-5.3-flash (crew frugal) · plan z-ai/glm-5.3-flash (crew frugal) ``` `codeaf do --json` carries the same four facts as `model`, `plan_model`, `model_source` diff --git a/internal/manual/pages/surfaces.md b/internal/manual/pages/surfaces.md index 558a7f0f25..4c802bca90 100644 --- a/internal/manual/pages/surfaces.md +++ b/internal/manual/pages/surfaces.md @@ -81,7 +81,9 @@ answer instead of picking. `esc` dismisses it. Everything you can tune lives in one sheet. Open it with `/settings`, the `⚙` in the header, or `alt+,` — a bare `,` works too whenever the cursor is not in the input. It is a single column of grouped rows: **models** -(all eight slots), **money & limits** (the daily budget, the practice +(all eight slots, plus the Model Pool row — `on` reads and sends, `read` +uses the pool and sends nothing, `off` does neither), **money & limits** +(the daily budget, the practice carve-out, the quiet period before practice), **rhythm** (how long an absence earns an arrival brief, how many clean firings earn a charter tenure), **learning** (how much practice follows measured demand rather than curiosity, diff --git a/internal/orchestrate/fuel.go b/internal/orchestrate/fuel.go index 2a3a035cbe..651a69feab 100644 --- a/internal/orchestrate/fuel.go +++ b/internal/orchestrate/fuel.go @@ -20,6 +20,7 @@ import ( "fmt" "strconv" "strings" + "sync/atomic" ) // The three answers to the gate. They are the whole vocabulary, and a surface @@ -43,6 +44,46 @@ type Price struct { Out float64 } +// PriceSource is a reader this package does not own, asked before the table +// falls back. It answers with the same pair PriceOf itself returns — one +// model's row in dollars per MILLION tokens — and false means "ask somebody +// else", never "free". +type PriceSource func(model string) (Price, bool) + +// priceTariff is the installed source, held behind an atomic because the +// package's meters read it on every call and the installer may swap it while a +// run is in flight. A nil tariff is the unset state: the table alone. +var priceTariff atomic.Value // PriceSource + +// UsePrices installs (or, with nil, removes) the source asked before the +// table. Replacing one source with another is the whole point — a tariff can +// move without this package being rebuilt — and a swap is safe to make while +// other goroutines are metering: a call in flight sees either the old source +// or the new one, never half of either. +func UsePrices(src PriceSource) { + priceTariff.Store(src) // a nil PriceSource is a stored value, not a nil store +} + +// CatalogPrices adapts a reader that answers per TOKEN — the shape a model +// catalog publishes — into the per-MILLION PriceSource this package meters +// with, so the unit conversion exists in exactly one place. A reader that +// knows a model carries the answer straight through, a per-token zero +// included; a reader that publishes nothing for it answers false, and the +// table gets its turn. A nil reader adapts to a nil source: nobody published +// anything. +func CatalogPrices(priceNow func(model string) (prompt, completion float64, known bool)) PriceSource { + if priceNow == nil { + return nil + } + return func(model string) (Price, bool) { + in, out, known := priceNow(model) + if !known { + return Price{}, false + } + return Price{In: in * 1e6, Out: out * 1e6}, true + } +} + // prices is the table this build meters against — the models an adaptive run // actually rides, at their OpenRouter list price. // @@ -52,12 +93,21 @@ type Price struct { // cannot know. [Meter] is what answers when a response carries no figure at // all, which is common enough that a run metering only reported costs would // have a tank that never empties. +// +// The rows are the ids the shipped defaults and crew tables name, each copied +// from the catalog's published price on the day its row was written; the +// installed tariff ([CatalogPrices]) is what answers when a catalog is present, +// and this table is only the fallback behind it. var prices = map[string]Price{ - "deepseek/deepseek-v4-flash": {In: 0.14, Out: 0.28}, - "z-ai/glm-5.2": {In: 0.60, Out: 2.20}, - "moonshotai/kimi-k3": {In: 0.60, Out: 2.50}, - "anthropic/claude-opus": {In: 15.00, Out: 75.00}, - "openai/gpt-5": {In: 1.25, Out: 10.00}, + "google/gemini-2.5-flash": {In: 0.30, Out: 2.50}, + "mistralai/mistral-nemo": {In: 0.02, Out: 0.03}, + "deepseek/deepseek-v4-flash-0731": {In: 0.06, Out: 0.12}, + "z-ai/glm-5.3-flash": {In: 0.09, Out: 0.30}, + "z-ai/glm-5.3": {In: 1.40, Out: 4.40}, + "moonshotai/kimi-k3": {In: 3.00, Out: 15.00}, + "qwen/qwen3.8-max-0902": {In: 2.00, Out: 6.00}, + "anthropic/claude-fable-5.1": {In: 10.00, Out: 50.00}, + "anthropic/claude-opus-5": {In: 5.00, Out: 25.00}, } // unpriced is what a model nobody has a row for costs. @@ -70,9 +120,11 @@ var prices = map[string]Price{ // governor people turn off. var unpriced = Price{In: 1.25, Out: 10.00} -// PriceOf is one model's row, and whether the table actually holds one. The id +// PriceOf is one model's row, and whether anybody actually holds one. The id // is matched the way the wire spells it, with suffixes an endpoint adds -// (":free", "@2026-01") cut before the lookup. +// (":free", "@2026-01") cut before the lookup — and the cut, lowercased name +// is what the installed source is asked too, so a source never has to repeat +// this package's normalisation to agree with its own table. // // A BARE NAME IS NOT A VENDOR'S ROW. A vendor serving a model on its own base // does not charge the router's price for it, and a wrong price is worse than no @@ -85,6 +137,11 @@ func PriceOf(model string) (Price, bool) { if name == "" { return unpriced, false } + if tariff, _ := priceTariff.Load().(PriceSource); tariff != nil { + if price, known := tariff(name); known { + return price, true + } + } if price, known := prices[name]; known { return price, true } diff --git a/internal/orchestrate/fuel_test.go b/internal/orchestrate/fuel_test.go index bce5a36113..3a01197dfa 100644 --- a/internal/orchestrate/fuel_test.go +++ b/internal/orchestrate/fuel_test.go @@ -10,17 +10,17 @@ import ( ) func TestMeter(t *testing.T) { - if got, want := MeterCall(1_000_000, 0, "openai/gpt-5"), 1.25; math.Abs(got-want) > 1e-9 { + if got, want := MeterCall(1_000_000, 0, "google/gemini-2.5-flash"), 0.30; math.Abs(got-want) > 1e-9 { t.Fatalf("input tokens: got %v want %v", got, want) } - if got, want := MeterCall(0, 1_000_000, "anthropic/claude-opus"), 75.0; math.Abs(got-want) > 1e-9 { + if got, want := MeterCall(0, 1_000_000, "anthropic/claude-fable-5.1"), 50.0; math.Abs(got-want) > 1e-9 { t.Fatalf("output tokens: got %v want %v", got, want) } // Meter has one number and charges it at the dear rate on purpose. - if Meter(1_000, "z-ai/glm-5.2") != MeterCall(0, 1_000, "z-ai/glm-5.2") { + if Meter(1_000, "z-ai/glm-5.3") != MeterCall(0, 1_000, "z-ai/glm-5.3") { t.Fatalf("Meter is the output rate") } - if _, known := PriceOf("z-ai/glm-5.2:free"); !known { + if _, known := PriceOf("z-ai/glm-5.3:free"); !known { t.Fatalf("an endpoint suffix is not a different model") } if _, known := PriceOf("kimi-k3"); known { @@ -285,3 +285,135 @@ func waitForEnd(t *testing.T, ended chan Snapshot) Snapshot { return Snapshot{} } } + +// TestInstalledSourceAnswersBeforeTheTable: the source is asked first, with +// the name this package normalised, and a source that knows the model is the +// answer even where the table holds a row of its own. +func TestInstalledSourceAnswersBeforeTheTable(t *testing.T) { + var asked []string + UsePrices(func(model string) (Price, bool) { + asked = append(asked, model) + if model == "z-ai/glm-5.3" { + return Price{In: 3, Out: 9}, true + } + return Price{}, false + }) + defer UsePrices(nil) + + price, known := PriceOf(" Z-Ai/GLM-5.3:free ") + if !known || price != (Price{In: 3, Out: 9}) { + t.Fatalf("the installed source is the answer: %+v, %v", price, known) + } + // THE SOURCE SEES THE NORMALISED NAME, not the raw one the caller passed: + // a source would otherwise have to repeat this package's trimming and + // suffix-cutting to agree with the table it sits above. + if len(asked) != 1 || asked[0] != "z-ai/glm-5.3" { + t.Fatalf("the source was asked %q", asked) + } +} + +// TestASourceThatDoesNotKnowFallsThroughToTheTable: false from the source is +// "ask somebody else", and the table's row for that model is what meters. +func TestASourceThatDoesNotKnowFallsThroughToTheTable(t *testing.T) { + UsePrices(func(string) (Price, bool) { return Price{}, false }) + defer UsePrices(nil) + + price, known := PriceOf("z-ai/glm-5.3") + if !known || price != (Price{In: 1.40, Out: 4.40}) { + t.Fatalf("a refusing source leaves the table in charge: %+v, %v", price, known) + } + if _, known := PriceOf("somebody/new-model"); known { + t.Fatalf("a refusing source does not invent a row") + } +} + +// TestUsePricesNilLeavesTheTableAlone: removing the source is the state the +// package started in. +func TestUsePricesNilLeavesTheTableAlone(t *testing.T) { + UsePrices(func(string) (Price, bool) { return Price{In: 99, Out: 99}, true }) + UsePrices(nil) + if price, known := PriceOf("moonshotai/kimi-k3"); !known || price != (Price{In: 3.00, Out: 15.00}) { + t.Fatalf("a removed source unmasks the table: %+v, %v", price, known) + } +} + +// TestCatalogPricesConvertsPerTokenToPerMillion: the catalog publishes per +// token, this package meters per million, and the doubling of that conversion +// stops here. +func TestCatalogPricesConvertsPerTokenToPerMillion(t *testing.T) { + if CatalogPrices(nil) != nil { + t.Fatalf("no reader is no source, not a source that never answers") + } + src := CatalogPrices(func(model string) (float64, float64, bool) { + if model == "vendored/big-model" { + return 0.0000035, 0.0000150, true + } + if model == "vendored/free-model" { + return 0, 0, true + } + return 0, 0, false + }) + price, known := src("vendored/big-model") + if !known || price != (Price{In: 3.5, Out: 15.0}) { + t.Fatalf("per-token became per-million: %+v, %v", price, known) + } + if price, known := src("vendored/free-model"); !known || (price != Price{}) { + t.Fatalf("a published zero is a published price: %+v, %v", price, known) + } + if _, known := src("vendored/unpublished"); known { + t.Fatalf("an unpublished model is not a free model") + } +} + +// TestTheMeterRidesTheInstalledSource: MeterCall prices through whatever is +// installed, so the whole tank — not just PriceOf — follows the tariff. +func TestTheMeterRidesTheInstalledSource(t *testing.T) { + UsePrices(CatalogPrices(func(model string) (float64, float64, bool) { + return 0.0000010, 0.0000030, true + })) + defer UsePrices(nil) + + if got, want := MeterCall(1_000_000, 1_000_000, "any/model"), 4.0; math.Abs(got-want) > 1e-9 { + t.Fatalf("the meter followed the source: got %v want %v", got, want) + } +} + +// TestRetiredIdsAreStrangers: an id the table no longer holds meters at the +// unpriced rate, exactly as one it never had. +func TestRetiredIdsAreStrangers(t *testing.T) { + for _, id := range []string{ + "z-ai/glm-5.2", "deepseek/deepseek-v4-flash", + "anthropic/claude-opus", "openai/gpt-5", + } { + price, known := PriceOf(id) + if known || price != unpriced { + t.Fatalf("PriceOf(%q) = %+v, %v; want %+v, false", id, price, known, unpriced) + } + } +} + +// TestMeteringSurvivesASwap: sources are swapped while runs meter, and every +// call sees one whole source. +func TestMeteringSurvivesASwap(t *testing.T) { + odd := func(string) (Price, bool) { return Price{In: 1, Out: 2}, true } + even := func(string) (Price, bool) { return Price{In: 2, Out: 4}, true } + UsePrices(odd) + defer UsePrices(nil) + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < 1000; i++ { + if i%2 == 0 { + UsePrices(odd) + } else { + UsePrices(even) + } + } + }() + for i := 0; i < 1000; i++ { + if _, known := PriceOf("any/model"); !known { + t.Fatalf("a swap lost the source mid-flight") + } + } + <-done +} diff --git a/internal/pool/index/index.go b/internal/pool/index/index.go new file mode 100644 index 0000000000..89c77b99c1 --- /dev/null +++ b/internal/pool/index/index.go @@ -0,0 +1,460 @@ +// Package index reads a schema-versioned JSON document of per-model +// measurements into a value every reader in a process can share. +// +// THE INDEX IS READ-ONLY AND IMMUTABLE, which is the whole design: nothing +// merges into it and nothing mutates it after Parse, so one document may be +// handed to every goroutine without a lock. Whoever read the file stays outside +// the package — Parse takes bytes, Read takes a reader, and Fallback chooses +// between two documents that both arrive as bytes. +// +// FORWARD COMPATIBILITY IS THE POINT. An unknown top-level field, an unknown +// metric kind, an undeclared dim on a cell and an unknown cell field are all +// ignored, because a reader that refused them would make every future field a +// flag day. The one thing refused outright is a schema this reader does not +// understand, which is what the schema number exists to say. +package index + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "strings" + "time" +) + +// ErrSchema is wrapped by the error a document whose schema is newer than +// this reader returns. A reader that is behind the document cannot guess what +// the newer fields mean, so it refuses rather than half-reads. +var ErrSchema = errors.New("document schema is newer than this reader") + +// Cell is one measurement from the document. Dims carries the declared dims +// beyond role and model that the cell spelled, and is nil for a cell that +// carried none. +type Cell struct { + Metric string + Role string + Model string + Mean float64 + SD float64 + N int + Dims map[string]string +} + +// Want is one row of the document's wanted list, with the role normalised and +// the model resolved to its canonical id. +type Want struct { + Role string + Model string + Weight float64 +} + +// Index is a parsed measurement document. Nothing on it mutates after it is +// built, and every accessor hands back copies, so many goroutines may read one +// Index at once. +type Index struct { + schema int + generated time.Time + minInstalls int + judges []string + rubrics map[string]int + // metrics holds the declared names as spelled; kinds and dims are keyed by + // the folded name every lookup arrives under. + metrics map[string]string + kinds map[string]string + dims map[string]map[string]bool + // aliases maps a folded, ~-stripped id to the canonical id as the document + // spells it. The canonical id is in the map under its own folded form. + aliases map[string]string + // cells is keyed by folded metric, then by an address that folds the role, + // the canonical model and the normalised dims into one exact key. + cells map[string]map[string]Cell + wanted []Want +} + +// Parse reads a measurement document from bytes. +func Parse(data []byte) (*Index, error) { + return parse(data) +} + +// Read reads a measurement document from a reader. The reader is drained and +// closed by nobody but the caller; the document is all this package takes. +func Read(r io.Reader) (*Index, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("read the index document: %w", err) + } + return parse(data) +} + +// Fallback answers with the document that should be trusted: primary, unless +// it is missing, empty, unparsable, or older by its generated date, in which +// case the embedded one. Equal dates keep primary. When neither parses it +// returns the error from the embedded document. +func Fallback(primary, embedded []byte) (*Index, error) { + p, perr := parse(primary) + e, eerr := parse(embedded) + if perr == nil { + if eerr != nil || !p.generated.Before(e.generated) { + return p, nil + } + return e, nil + } + if eerr != nil { + return nil, eerr + } + return e, nil +} + +// document is the wire shape. Fields this build does not know are dropped by +// the decoder, which is the forward-compatibility rule above. +type document struct { + Schema *int `json:"schema"` + Generated string `json:"generated"` + MinInstalls int `json:"min_installs"` + Judges []string `json:"judges"` + Rubrics map[string]int `json:"rubrics"` + Aliases map[string][]string `json:"aliases"` + Metrics map[string]metricDecl `json:"metrics"` + Cells []map[string]any `json:"cells"` + Wanted []wantEntry `json:"wanted"` +} + +type metricDecl struct { + Kind string `json:"kind"` + Dims []string `json:"dims"` +} + +type wantEntry struct { + Role string `json:"role"` + Model string `json:"model"` + Weight float64 `json:"weight"` +} + +// reservedCellFields are the fixed fields of a cell. A declared dim named one +// of these could never be read off a cell, so the name is reserved rather than +// resolved twice. +var reservedCellFields = map[string]bool{ + "metric": true, "role": true, "model": true, "mean": true, "sd": true, "n": true, +} + +// The separators are unprintable so no dim key or value can contain one and +// forge a different address. +const ( + pairSep = "\x1f" + keySep = "\x1e" +) + +func parse(data []byte) (*Index, error) { + if len(bytes.TrimSpace(data)) == 0 { + return nil, errors.New("index: empty document") + } + var d document + if err := json.Unmarshal(data, &d); err != nil { + return nil, fmt.Errorf("index: decode: %w", err) + } + if d.Schema != nil && *d.Schema > 1 { + return nil, fmt.Errorf("index: schema %d is newer than this reader, which understands 1: %w", *d.Schema, ErrSchema) + } + + x := &Index{ + schema: 0, + minInstalls: d.MinInstalls, + judges: d.Judges, + rubrics: d.Rubrics, + metrics: map[string]string{}, + kinds: map[string]string{}, + dims: map[string]map[string]bool{}, + aliases: map[string]string{}, + cells: map[string]map[string]Cell{}, + } + if d.Schema != nil { + x.schema = *d.Schema + } + if stamp, err := time.Parse("2006-01-02", d.Generated); err == nil { + x.generated = stamp.UTC() + } + + // The declared metrics are keyed by their folded name for lookup, and + // their kinds are folded once here rather than on every Kind call. The + // names are walked in sorted order for the same reason the aliases below + // are: two spellings that fold to one name are a first-claim-wins race, + // and map order would settle it differently on every run. + metricNames := make([]string, 0, len(d.Metrics)) + for name := range d.Metrics { + metricNames = append(metricNames, name) + } + sort.Strings(metricNames) + for _, name := range metricNames { + decl := d.Metrics[name] + folded := fold(name) + if _, seen := x.metrics[folded]; !seen { + x.metrics[folded] = name + } + if _, seen := x.kinds[folded]; !seen { + x.kinds[folded] = fold(decl.Kind) + } + dimset := map[string]bool{} + for _, dim := range decl.Dims { + fd := fold(dim) + if fd == "role" || fd == "model" { + continue + } + dimset[fd] = true + } + if _, seen := x.dims[folded]; !seen { + x.dims[folded] = dimset + } + } + + // Aliases are resolved in sorted canonical order so a name claimed by two + // canonical ids always resolves the same way. + ids := make([]string, 0, len(d.Aliases)) + for id := range d.Aliases { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + x.addAlias(id, id) + for _, alt := range d.Aliases[id] { + x.addAlias(alt, id) + } + } + + for _, raw := range d.Cells { + x.addCell(raw) + } + + for _, w := range d.Wanted { + x.wanted = append(x.wanted, Want{Role: fold(w.Role), Model: x.resolve(w.Model), Weight: w.Weight}) + } + sort.SliceStable(x.wanted, func(i, j int) bool { + if x.wanted[i].Weight != x.wanted[j].Weight { + return x.wanted[i].Weight > x.wanted[j].Weight + } + return x.wanted[i].Model < x.wanted[j].Model + }) + return x, nil +} + +// addAlias records that id is an alternate spelling of canonical, unless the +// id already resolves — the first claim in sorted order wins. +func (x *Index) addAlias(id, canonical string) { + key := normModel(id) + if _, seen := x.aliases[key]; !seen { + x.aliases[key] = canonical + } +} + +// addCell reads one raw cell into the index, or drops it. A cell whose metric +// is not declared is skipped — a measurement with no metric behind it has no +// units, and guessing is worse than dropping it. A cell whose N is below the +// document's min_installs is skipped too: too few observations is not a +// measurement. +func (x *Index) addCell(raw map[string]any) { + metricName, _ := raw["metric"].(string) + mk := fold(metricName) + if _, declared := x.metrics[mk]; !declared { + return + } + role, _ := raw["role"].(string) + model, _ := raw["model"].(string) + n := int(number(raw["n"])) + if n < x.minInstalls { + return + } + dims := map[string]string{} + for k, v := range raw { + if reservedCellFields[fold(k)] { + continue + } + if !x.dims[mk][fold(k)] { + continue + } + if s, ok := v.(string); ok { + dims[k] = s + } + } + canon := x.resolve(model) + c := Cell{ + Metric: metricName, + Role: role, + Model: canon, + Mean: number(raw["mean"]), + SD: number(raw["sd"]), + N: n, + } + if len(dims) > 0 { + c.Dims = dims + } + bucket := x.cells[mk] + if bucket == nil { + bucket = map[string]Cell{} + x.cells[mk] = bucket + } + bucket[address(role, canon, c.Dims)] = c +} + +// address renders the exact place a measurement lives: folded role, folded +// canonical model, and the normalised dims as sorted key-value pairs. A nil +// dims map and an empty one render the same, which is the rule — they are the +// same address. +func address(role, canonicalModel string, dims map[string]string) string { + parts := []string{fold(role), fold(canonicalModel)} + if len(dims) > 0 { + pairs := make([]string, 0, len(dims)) + for k, v := range dims { + pairs = append(pairs, fold(k)+keySep+fold(v)) + } + sort.Strings(pairs) + parts = append(parts, strings.Join(pairs, pairSep)) + } + return strings.Join(parts, pairSep) +} + +// resolve answers with the canonical id a model id is an alternate of, or the +// normalised id itself when the document never names it. +func (x *Index) resolve(model string) string { + key := normModel(model) + if canon, ok := x.aliases[key]; ok { + return canon + } + return key +} + +// fold is how metric names, roles, dim keys and dim values are matched. +func fold(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} + +// normModel folds a model id and then strips one leading "~". +func normModel(s string) string { + return strings.TrimPrefix(fold(s), "~") +} + +// number reads a JSON number, and zero for anything else on the cell — a +// missing field is not a malformed document. +func number(v any) float64 { + f, _ := v.(float64) + return f +} + +// Schema is the document's schema version. A missing schema reads as 0 and is +// accepted. +func (x *Index) Schema() int { return x.schema } + +// Generated is the document's "generated" day in UTC. A missing or unreadable +// date is the zero time, which sorts as the oldest thing there is. +func (x *Index) Generated() time.Time { return x.generated } + +// MinInstalls is the document's floor on observations per measurement. +func (x *Index) MinInstalls() int { return x.minInstalls } + +// Judges answers with a copy of the document's judges. +func (x *Index) Judges() []string { + if x.judges == nil { + return nil + } + return append([]string(nil), x.judges...) +} + +// Rubrics answers with a copy of the document's rubrics. +func (x *Index) Rubrics() map[string]int { + out := make(map[string]int, len(x.rubrics)) + for k, v := range x.rubrics { + out[k] = v + } + return out +} + +// Metrics lists the declared metric names, sorted. +func (x *Index) Metrics() []string { + out := make([]string, 0, len(x.metrics)) + for _, name := range x.metrics { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// Kind answers with the kind word the document spells for the metric, +// lowercased, whether or not this build knows it. +func (x *Index) Kind(metric string) (string, bool) { + kind, ok := x.kinds[fold(metric)] + return kind, ok +} + +// Canonical answers with the canonical id the document spells for a model id, +// for the id itself and for any of its alternates, and with the normalised +// input for an id the document never names. +func (x *Index) Canonical(model string) string { + return x.resolve(model) +} + +// Cell addresses a measurement exactly: metric, role, model and dims all have +// to match, and a nil dims map and an empty one are the same address. A lookup +// with no dims finds the cell that carries none, and never a cell that carries +// some. +func (x *Index) Cell(metric, role, model string, dims map[string]string) (Cell, bool) { + bucket := x.cells[fold(metric)] + if bucket == nil { + return Cell{}, false + } + c, ok := bucket[address(role, x.resolve(model), dims)] + if !ok { + return Cell{}, false + } + return copyCell(c), true +} + +// Cells answers with every returnable cell of one metric, sorted by role, then +// model, then by its dims rendered as sorted "key=value" pairs. +func (x *Index) Cells(metric string) []Cell { + bucket := x.cells[fold(metric)] + out := make([]Cell, 0, len(bucket)) + for _, c := range bucket { + out = append(out, copyCell(c)) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Role != out[j].Role { + return out[i].Role < out[j].Role + } + if out[i].Model != out[j].Model { + return out[i].Model < out[j].Model + } + return dimsKey(out[i].Dims) < dimsKey(out[j].Dims) + }) + return out +} + +// Wanted answers with a copy of the wanted list, sorted by weight descending, +// then by model ascending. +func (x *Index) Wanted() []Want { + return append([]Want(nil), x.wanted...) +} + +// dimsKey renders a cell's dims for sorting: sorted "key=value" pairs. +func dimsKey(dims map[string]string) string { + pairs := make([]string, 0, len(dims)) + for k, v := range dims { + pairs = append(pairs, k+"="+v) + } + sort.Strings(pairs) + return strings.Join(pairs, ",") +} + +// copyCell hands back a cell whose Dims map is its own, so a caller cannot +// reach into the shared document. +func copyCell(c Cell) Cell { + if c.Dims == nil { + return c + } + dims := make(map[string]string, len(c.Dims)) + for k, v := range c.Dims { + dims[k] = v + } + c.Dims = dims + return c +} diff --git a/internal/pool/index/index_test.go b/internal/pool/index/index_test.go new file mode 100644 index 0000000000..8ebe165f4e --- /dev/null +++ b/internal/pool/index/index_test.go @@ -0,0 +1,519 @@ +package index + +import ( + "bytes" + "errors" + "strings" + "sync" + "testing" + "time" +) + +// example is the shape the whole reader holds in mind: one metric with dims, +// one alias pair, one cell, one wanted row. +const example = `{ + "schema": 1, + "generated": "2026-09-17", + "min_installs": 5, + "judges": ["z-ai/glm-5.3"], + "rubrics": {"planner": 3}, + "aliases": {"z-ai/glm-5.3": ["glm-5.3", "zai-org/GLM-5.3"]}, + "metrics": {"role_rating": {"kind": "gaussian", "unit": "elo", "dims": ["role", "model"]}}, + "cells": [ + {"metric": "role_rating", "role": "planner", "model": "z-ai/glm-5.3", "mean": 1312, "sd": 18, "n": 410} + ], + "wanted": [{"role": "planner", "model": "qwen/qwen3.8-max-0902", "weight": 0.4}] +}` + +func mustParse(t *testing.T, data string) *Index { + t.Helper() + x, err := Parse([]byte(data)) + if err != nil { + t.Fatalf("parse: %v", err) + } + return x +} + +// ── THE DOCUMENT, READ BACK ───────────────────────────────────────────────── + +func TestTheDocumentReadsBack(t *testing.T) { + x := mustParse(t, example) + if x.Schema() != 1 { + t.Errorf("schema is %d, want 1", x.Schema()) + } + if got, want := x.Generated(), time.Date(2026, 9, 17, 0, 0, 0, 0, time.UTC); !got.Equal(want) { + t.Errorf("generated is %v, want %v", got, want) + } + if x.MinInstalls() != 5 { + t.Errorf("min_installs is %d, want 5", x.MinInstalls()) + } + if judges := x.Judges(); len(judges) != 1 || judges[0] != "z-ai/glm-5.3" { + t.Errorf("judges are %v", judges) + } + if rubrics := x.Rubrics(); rubrics["planner"] != 3 || len(rubrics) != 1 { + t.Errorf("rubrics are %v", rubrics) + } + if metrics := x.Metrics(); len(metrics) != 1 || metrics[0] != "role_rating" { + t.Errorf("metrics are %v", metrics) + } + if kind, ok := x.Kind("role_rating"); !ok || kind != "gaussian" { + t.Errorf("kind is %q, %v", kind, ok) + } + c, ok := x.Cell("role_rating", "planner", "z-ai/glm-5.3", nil) + if !ok { + t.Fatal("the example cell was not found") + } + if c.Mean != 1312 || c.SD != 18 || c.N != 410 { + t.Errorf("cell reads %v", c) + } + if len(x.Wanted()) != 1 || x.Wanted()[0].Weight != 0.4 { + t.Errorf("wanted reads %v", x.Wanted()) + } +} + +func TestReadTakesAReader(t *testing.T) { + x, err := Read(bytes.NewBufferString(example)) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(x.Cells("role_rating")) != 1 { + t.Errorf("read through a reader found %d cells", len(x.Cells("role_rating"))) + } +} + +// ── FORWARD COMPATIBILITY ─────────────────────────────────────────────────── + +// EVERY UNKNOWN THING IS IGNORED, because a reader that refused them would +// make every future field a flag day. +func TestUnknownThingsAreIgnoredNotErrors(t *testing.T) { + x := mustParse(t, `{ + "schema": 1, + "tomorrow_field": {"anything": true}, + "metrics": {"role_rating": {"kind": "not-a-kind-we-know", "dims": []}}, + "cells": [ + {"metric": "role_rating", "role": "planner", "model": "m", "mean": 1, "sd": 1, "n": 9, "unknown_cell_field": "x"} + ] + }`) + if kind, ok := x.Kind("role_rating"); !ok || kind != "not-a-kind-we-know" { + t.Errorf("an unknown kind was refused or rewritten: %q %v", kind, ok) + } + if _, ok := x.Cell("role_rating", "planner", "m", nil); !ok { + t.Error("a cell carrying an unknown field was dropped") + } +} + +func TestSchemaRefusesOnlyWhatIsTooNew(t *testing.T) { + for _, row := range []struct { + name, schema string + want int + accepted bool + }{ + {"a missing schema", "", 0, true}, // missing reads as 0 and is accepted + {"schema 0", `"schema": 0,`, 0, true}, + {"schema 1", `"schema": 1,`, 1, true}, + {"a schema below 1", `"schema": -1,`, -1, true}, + {"schema 2", `"schema": 2,`, 0, false}, + {"schema 99", `"schema": 99,`, 0, false}, + } { + x, err := Parse([]byte("{" + row.schema + `"metrics": {}}`)) + if (err == nil) != row.accepted { + t.Errorf("%s: err=%v, want accepted=%v", row.name, err, row.accepted) + continue + } + if row.accepted && x.Schema() != row.want { + t.Errorf("%s reads back as %d, want %d", row.name, x.Schema(), row.want) + } + } + + _, err := Parse([]byte(`{"schema": 2}`)) + if !errors.Is(err, ErrSchema) { + t.Errorf("a too-new schema answered %v, want it wrapped in ErrSchema", err) + } +} + +func TestMalformedAndEmptyDocumentsAreErrors(t *testing.T) { + for _, row := range []struct { + name string + data string + }{ + {"malformed json", `{"schema": 1,`}, + {"an empty document", ``}, + {"a whitespace document", " \n\t "}, + {"a document that is not an object", `[1, 2, 3]`}, + } { + if _, err := Parse([]byte(row.data)); err == nil { + t.Errorf("%s was accepted", row.name) + } + } +} + +func TestAGeneratedDateThatWillNotReadIsTheZeroTime(t *testing.T) { + for _, row := range []struct { + name, generated string + wantZero bool + }{ + {"missing", "", true}, + {"unreadable", "17/09/2026", true}, + {"readable", "2026-09-17", false}, + } { + data := `{"metrics": {}}` + if row.generated != "" { + data = `{"generated": "` + row.generated + `", "metrics": {}}` + } + got := mustParse(t, data).Generated() + if row.wantZero != got.IsZero() { + t.Errorf("%s generated date gave %v, want zero=%v", row.name, got, row.wantZero) + } + } +} + +// ── METRICS AND DIMS ──────────────────────────────────────────────────────── + +func TestMetricsAreSortedAndCellsOfUndeclaredMetricsAreSkipped(t *testing.T) { + x := mustParse(t, `{ + "metrics": {"zebra": {"kind": "a"}, "alpha": {"kind": "b"}, "mid": {"kind": "c"}}, + "cells": [ + {"metric": "zebra", "role": "r", "model": "m", "mean": 1, "sd": 1, "n": 9}, + {"metric": "undeclared", "role": "r", "model": "m", "mean": 1, "sd": 1, "n": 9} + ] + }`) + if got, want := strings.Join(x.Metrics(), ","), "alpha,mid,zebra"; got != want { + t.Errorf("metrics are %q, want %q", got, want) + } + if len(x.Cells("zebra")) != 1 { + t.Errorf("a declared metric's cell count is %d, want 1", len(x.Cells("zebra"))) + } + if len(x.Cells("undeclared")) != 0 { + t.Error("a cell whose metric is not declared was kept") + } + if _, ok := x.Kind("undeclared"); ok { + t.Error("an undeclared metric answered a kind") + } +} + +func TestDeclaredDimsBecomeCellDims(t *testing.T) { + x := mustParse(t, `{ + "metrics": {"m": {"kind": "a", "dims": ["role", "model", "region", "variant"]}}, + "cells": [ + {"metric": "m", "role": "r", "model": "mo", "mean": 1, "sd": 1, "n": 1, + "region": "EU", "variant": "lite", "mean_x": "not a dim", "n_other": 7} + ] + }`) + c, ok := x.Cell("m", "r", "mo", map[string]string{"region": "eu", "variant": "LITE"}) + if !ok { + t.Fatal("the dims cell was not found") + } + if len(c.Dims) != 2 || c.Dims["region"] != "EU" || c.Dims["variant"] != "lite" { + t.Errorf("dims read %v, want region=EU and variant=lite as spelled", c.Dims) + } + // role and model are declared dims but never cell dims — they are the + // address itself. + for _, banned := range []string{"role", "model"} { + if _, is := c.Dims[banned]; is { + t.Errorf("%s became a cell dim", banned) + } + } +} + +func TestADimWhoseValueIsNotAStringIsIgnored(t *testing.T) { + x := mustParse(t, `{ + "metrics": {"m": {"kind": "a", "dims": ["region"]}}, + "cells": [ + {"metric": "m", "role": "r", "model": "mo", "mean": 1, "sd": 1, "n": 1, "region": 4} + ] + }`) + c, ok := x.Cell("m", "r", "mo", nil) + if !ok { + t.Fatal("the cell was dropped over a non-string dim") + } + if len(c.Dims) != 0 { + t.Errorf("a non-string dim became %v", c.Dims) + } +} + +// ── NORMALISATION AND ALIASES ─────────────────────────────────────────────── + +// THE SAME ID CAN ARRIVE SPELLED FOUR WAYS, and a lookup written with one of +// them has to find a cell written with another: that is the only way an +// alias table earns its place in the document. +func TestAliasesMeetInBothDirections(t *testing.T) { + x := mustParse(t, `{ + "aliases": {"Z-AI/GLM-5.3": ["glm-5.3"]}, + "metrics": {"m": {"kind": "a"}}, + "cells": [{"metric": "m", "role": "planner", "model": "glm-5.3", "mean": 2, "sd": 1, "n": 5}], + "wanted": [{"role": " Planner ", "model": " glm-5.3 ", "weight": 1}] + }`) + if got, want := x.Canonical("~z-ai/glm-5.3"), "Z-AI/GLM-5.3"; got != want { + t.Errorf("canonical of the folded id is %q, want %q", got, want) + } + if got, want := x.Canonical("~glm-5.3"), "Z-AI/GLM-5.3"; got != want { + t.Errorf("canonical of the ~-stripped alias is %q, want %q", got, want) + } + if got, want := x.Canonical("never/named"), "never/named"; got != want { + t.Errorf("canonical of an unnamed id is %q, want the normalised input %q", got, want) + } + // The cell was written under an alias; a lookup under a different alias + // finds it because cells are stored under their canonical model. + c, ok := x.Cell(" M ", "PLANNER", "~Z-AI/GLM-5.3", nil) + if !ok { + t.Fatal("a lookup under one alias did not find a cell written under another") + } + if c.Model != "Z-AI/GLM-5.3" { + t.Errorf("the cell's model is %q, want the canonical id", c.Model) + } + if w := x.Wanted(); len(w) != 1 || w[0].Role != "planner" || w[0].Model != "Z-AI/GLM-5.3" { + t.Errorf("wanted reads %v, want the role folded and the model canonical", w) + } +} + +func TestMetricNamesRolesAndDimsMatchFolded(t *testing.T) { + x := mustParse(t, `{ + "metrics": {"Role_Rating": {"kind": "a", "dims": [" Region "]}}, + "cells": [{"metric": "role_rating", "role": " Planner ", "model": "m", "mean": 1, "sd": 1, "n": 1, "region": "eu"}] + }`) + if _, ok := x.Kind(" ROLE_RATING "); !ok { + t.Error("a kind lookup did not fold the metric name") + } + if _, ok := x.Cell("ROLE_rating", "planner", "M", map[string]string{"REGION": "EU"}); !ok { + t.Error("an exact address did not fold metric, role and dims") + } +} + +// ── THE EXACT ADDRESS ─────────────────────────────────────────────────────── + +// NIL DIMS AND EMPTY DIMS ARE THE SAME ADDRESS, and a lookup that carries +// some dims never finds a cell that carries none, nor the other way round. +func TestTheAddressIsExact(t *testing.T) { + x := mustParse(t, `{ + "metrics": {"m": {"kind": "a", "dims": ["region", "tier"]}}, + "cells": [ + {"metric": "m", "role": "r", "model": "a", "mean": 1, "sd": 1, "n": 1}, + {"metric": "m", "role": "r", "model": "b", "mean": 2, "sd": 1, "n": 1, "region": "eu"}, + {"metric": "m", "role": "r", "model": "c", "mean": 3, "sd": 1, "n": 1, "region": "eu", "tier": "lite"} + ] + }`) + if c, ok := x.Cell("m", "r", "a", map[string]string{}); !ok || c.Mean != 1 { + t.Error("an empty dims map did not find the cell that carries none") + } + if _, ok := x.Cell("m", "r", "b", nil); ok { + t.Error("a no-dims lookup found a cell that carries dims") + } + if c, ok := x.Cell("m", "r", "b", map[string]string{"region": "eu"}); !ok || c.Mean != 2 { + t.Error("the one-dim cell was not found by its one dim") + } + if _, ok := x.Cell("m", "r", "c", map[string]string{"region": "eu"}); ok { + t.Error("a partial dims lookup found a cell with more dims") + } + if _, ok := x.Cell("m", "r", "b", map[string]string{"region": "us"}); ok { + t.Error("a dims lookup with the wrong value found the cell") + } +} + +func TestMinInstallsHoldsBackThinCells(t *testing.T) { + x := mustParse(t, `{ + "min_installs": 5, + "metrics": {"m": {"kind": "a"}}, + "cells": [ + {"metric": "m", "role": "r", "model": "thin", "mean": 1, "sd": 1, "n": 4}, + {"metric": "m", "role": "r", "model": "held", "mean": 2, "sd": 1, "n": 5} + ] + }`) + if _, ok := x.Cell("m", "r", "thin", nil); ok { + t.Error("a cell below min_installs was returned by Cell") + } + names := "" + for _, c := range x.Cells("m") { + names += c.Model + } + if names != "held" { + t.Errorf("Cells answered %q, want only the cell that met min_installs", names) + } +} + +func TestCellsAreSortedByRoleThenModelThenDims(t *testing.T) { + x := mustParse(t, `{ + "metrics": {"m": {"kind": "a", "dims": ["region", "tier"]}}, + "cells": [ + {"metric": "m", "role": "zebra", "model": "a", "mean": 0, "sd": 0, "n": 1}, + {"metric": "m", "role": "alpha", "model": "z", "mean": 0, "sd": 0, "n": 1}, + {"metric": "m", "role": "alpha", "model": "a", "mean": 0, "sd": 0, "n": 1}, + {"metric": "m", "role": "alpha", "model": "a", "mean": 0, "sd": 0, "n": 1, "region": "eu", "tier": "lite"}, + {"metric": "m", "role": "alpha", "model": "a", "mean": 0, "sd": 0, "n": 1, "region": "eu"}, + {"metric": "m", "role": "alpha", "model": "a", "mean": 0, "sd": 0, "n": 1, "tier": "lite"} + ] + }`) + got := x.Cells("m") + order := "" + for _, c := range got { + order += c.Role + ":" + c.Model + ":" + dimsKey(c.Dims) + ";" + } + want := "alpha:a:;alpha:a:region=eu;alpha:a:region=eu,tier=lite;alpha:a:tier=lite;alpha:z:;zebra:a:;" + if order != want { + t.Errorf("cells came out as\n%s\nwant\n%s", order, want) + } +} + +func TestWantedIsSortedByWeightThenModel(t *testing.T) { + x := mustParse(t, `{ + "aliases": {"c/one": []}, + "wanted": [ + {"role": "r", "model": "c/one", "weight": 0.5}, + {"role": "r", "model": "a/two", "weight": 0.5}, + {"role": "r", "model": "b/three", "weight": 0.9} + ] + }`) + got := x.Wanted() + order := "" + for _, w := range got { + order += w.Model + "," + } + if order != "b/three,a/two,c/one," { + t.Errorf("wanted came out as %q", order) + } +} + +// ── COPIES AND SHARING ────────────────────────────────────────────────────── + +// NOTHING A CALLER HOLDS REACHES INTO THE SHARED DOCUMENT: every accessor +// hands back copies, and a mutation through one must not leak into another. +func TestReturnedValuesAreCopies(t *testing.T) { + x := mustParse(t, example) + + judges := x.Judges() + judges[0] = "mutated" + if x.Judges()[0] == "mutated" { + t.Error("Judges handed back the shared slice") + } + + rubrics := x.Rubrics() + rubrics["planner"] = 99 + if x.Rubrics()["planner"] == 99 { + t.Error("Rubrics handed back the shared map") + } + + metrics := x.Metrics() + metrics[0] = "mutated" + if x.Metrics()[0] == "mutated" { + t.Error("Metrics handed back the shared slice") + } + + c, _ := x.Cell("role_rating", "planner", "z-ai/glm-5.3", nil) + if c.Dims != nil { + c.Dims["region"] = "mutated" + } + cells := x.Cells("role_rating") + if len(cells) == 1 && cells[0].Dims != nil { + cells[0].Dims["region"] = "mutated" + if again, _ := x.Cell("role_rating", "planner", "z-ai/glm-5.3", nil); again.Dims != nil && again.Dims["region"] == "mutated" { + t.Error("Cells handed back the shared dims map") + } + } + + wanted := x.Wanted() + wanted[0].Weight = -1 + if x.Wanted()[0].Weight == -1 { + t.Error("Wanted handed back the shared slice") + } +} + +func TestDimsMapsAreCopiesOnEveryCell(t *testing.T) { + x := mustParse(t, `{ + "metrics": {"m": {"kind": "a", "dims": ["region"]}}, + "cells": [{"metric": "m", "role": "r", "model": "a", "mean": 1, "sd": 1, "n": 1, "region": "eu"}] + }`) + first, _ := x.Cell("m", "r", "a", map[string]string{"region": "eu"}) + first.Dims["region"] = "mutated" + second, _ := x.Cell("m", "r", "a", map[string]string{"region": "eu"}) + if second.Dims["region"] == "mutated" { + t.Error("a dims map handed back by Cell was the shared one") + } +} + +// ONE INDEX, MANY GOROUTINES, NO LOCK: the immutability is the design, and +// this is the cheapest way to catch anything that quietly broke it. +func TestOneIndexServesManyReadersAtOnce(t *testing.T) { + x := mustParse(t, `{ + "min_installs": 2, + "metrics": {"m": {"kind": "a", "dims": ["region"]}}, + "cells": [ + {"metric": "m", "role": "r", "model": "a", "mean": 1, "sd": 1, "n": 9, "region": "eu"} + ], + "wanted": [{"role": "r", "model": "a", "weight": 1}] + }`) + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 200; i++ { + x.Metrics() + x.Kind("m") + x.Cells("m") + x.Cell("m", "r", "a", map[string]string{"region": "eu"}) + x.Wanted() + x.Judges() + x.Rubrics() + x.Canonical("a") + } + }() + } + wg.Wait() +} + +// ── FALLBACK ──────────────────────────────────────────────────────────────── + +const fallbackEmbedded = `{"schema": 1, "generated": "2026-09-01", "metrics": {"old": {"kind": "a"}}}` + +func TestFallbackPrefersPrimaryAndFallsBackOnItsFailures(t *testing.T) { + fresh := `{"schema": 1, "generated": "2026-09-17", "metrics": {"fresh": {"kind": "a"}}}` + for _, row := range []struct { + name, primary string + wantMetric string + }{ + {"a fresh primary", fresh, "fresh"}, + {"a missing primary", "", "old"}, + {"an empty primary", " ", "old"}, + {"an unparsable primary", `{"schema": `, "old"}, + {"a too-new primary", `{"schema": 2}`, "old"}, + {"an older primary", `{"schema": 1, "generated": "2026-08-31", "metrics": {"stale": {"kind": "a"}}}`, "old"}, + } { + x, err := Fallback([]byte(row.primary), []byte(fallbackEmbedded)) + if err != nil { + t.Errorf("%s: %v", row.name, err) + continue + } + if got := x.Metrics(); len(got) != 1 || got[0] != row.wantMetric { + t.Errorf("%s answered with %v, want the %s document", row.name, got, row.wantMetric) + } + } +} + +func TestFallbackKeepsPrimaryOnEqualDates(t *testing.T) { + x, err := Fallback( + []byte(`{"schema": 1, "generated": "2026-09-17", "metrics": {"primary": {"kind": "a"}}}`), + []byte(fallbackEmbedded), + ) + if err != nil { + t.Fatal(err) + } + if got := x.Metrics(); len(got) != 1 || got[0] != "primary" { + t.Errorf("equal dates answered with %v, want primary kept", got) + } +} + +func TestFallbackKeepsPrimaryWhenEmbeddedWillNotParse(t *testing.T) { + x, err := Fallback([]byte(`{"metrics": {"primary": {"kind": "a"}}}`), []byte("not json")) + if err != nil { + t.Fatal(err) + } + if got := x.Metrics(); len(got) != 1 || got[0] != "primary" { + t.Errorf("a good primary lost to a broken embedded: %v", got) + } +} + +func TestFallbackReturnsTheEmbeddedErrorWhenNeitherParses(t *testing.T) { + _, err := Fallback([]byte(""), []byte(`{"schema": `)) + if err == nil || !strings.Contains(err.Error(), "decode") { + t.Errorf("neither parsing answered %v, want the embedded document's error", err) + } +} diff --git a/internal/pool/index/seed.go b/internal/pool/index/seed.go new file mode 100644 index 0000000000..126c92737a --- /dev/null +++ b/internal/pool/index/seed.go @@ -0,0 +1,33 @@ +package index + +import ( + _ "embed" +) + +// seedJSON is the index this build carries: the first measured pool, shipped +// inside the binary so a machine that has never fetched one still has numbers +// to pick a seat against on its first run. +// +// IT IS A FALLBACK AND NOT A SOURCE. Whoever reads an index reads the cache +// first and this beside it ([Fallback]): a fresh signed document wins, and +// this answers only when there is none, or when the one on disk is older or +// unreadable. seed.json is the same bytes as a file under version control, so +// what the binary carries is diffed and reviewed like any other figure here. +// +//go:embed seed.json +var seedJSON []byte + +// Seed answers with the embedded document's bytes, as a COPY: the index is +// read many times and by many goroutines, and a caller that changed the slice +// would change what every later [Seed] call hands back. +func Seed() []byte { + return append([]byte(nil), seedJSON...) +} + +// SeedIndex parses the embedded document — the index a machine with no cache +// starts from. The error is returned rather than panicking, though the seed is +// a file this build was shipped with and a test in this package parses it, so +// an error here is a bug and not a mode. +func SeedIndex() (*Index, error) { + return Parse(seedJSON) +} diff --git a/internal/pool/index/seed.json b/internal/pool/index/seed.json new file mode 100644 index 0000000000..af53d24ab7 --- /dev/null +++ b/internal/pool/index/seed.json @@ -0,0 +1,148 @@ +{ + "schema": 1, + "generated": "2026-09-17", + "min_installs": 1, + "judges": [ + "reviewer" + ], + "rubrics": { + "role_quality": 1 + }, + "aliases": { + "deepseek/deepseek-v4.1-flash": [ + "ds-v4.1-flash" + ] + }, + "metrics": { + "role_quality": { + "kind": "gaussian", + "unit": "score", + "dims": [ + "role", + "model" + ] + } + }, + "cells": [ + { + "metric": "role_quality", + "role": "high", + "model": "anthropic/claude-fable-5.1", + "mean": 87.1, + "sd": 5.9, + "n": 29 + }, + { + "metric": "role_quality", + "role": "high", + "model": "anthropic/claude-opus-5", + "mean": 83.3, + "sd": 14.1, + "n": 4 + }, + { + "metric": "role_quality", + "role": "high", + "model": "moonshotai/kimi-k3", + "mean": 86.2, + "sd": 6.3, + "n": 27 + }, + { + "metric": "role_quality", + "role": "high", + "model": "qwen/qwen3.8-max-0902", + "mean": 92.0, + "sd": 5.3, + "n": 23 + }, + { + "metric": "role_quality", + "role": "mastermind", + "model": "anthropic/claude-fable-5.1", + "mean": 77.5, + "sd": 6.5, + "n": 38 + }, + { + "metric": "role_quality", + "role": "mastermind", + "model": "anthropic/claude-opus-5", + "mean": 81.0, + "sd": 8.4, + "n": 19 + }, + { + "metric": "role_quality", + "role": "mastermind", + "model": "moonshotai/kimi-k3", + "mean": 57.1, + "sd": 10.6, + "n": 19 + }, + { + "metric": "role_quality", + "role": "mastermind", + "model": "qwen/qwen3.8-max-0902", + "mean": 50.0, + "sd": 16.7, + "n": 6 + }, + { + "metric": "role_quality", + "role": "mastermind", + "model": "z-ai/glm-5.3", + "mean": 62.5, + "sd": 16.1, + "n": 6 + }, + { + "metric": "role_quality", + "role": "worker", + "model": "deepseek/deepseek-v4.1-flash", + "mean": 83.3, + "sd": 6.1, + "n": 34 + }, + { + "metric": "role_quality", + "role": "worker", + "model": "meta/muse-spark-1.3", + "mean": 94.1, + "sd": 5.5, + "n": 15 + }, + { + "metric": "role_quality", + "role": "worker", + "model": "openai/gpt-5.6-sol", + "mean": 85.7, + "sd": 12.4, + "n": 5 + }, + { + "metric": "role_quality", + "role": "worker", + "model": "qwen/qwen3.8-max-0902", + "mean": 72.7, + "sd": 12.9, + "n": 9 + }, + { + "metric": "role_quality", + "role": "worker", + "model": "z-ai/glm-5.3", + "mean": 72.5, + "sd": 7.0, + "n": 38 + }, + { + "metric": "role_quality", + "role": "worker", + "model": "z-ai/glm-5.3-flash", + "mean": 67.4, + "sd": 4.9, + "n": 90 + } + ] +} diff --git a/internal/pool/index/seed_test.go b/internal/pool/index/seed_test.go new file mode 100644 index 0000000000..b6ae57e35f --- /dev/null +++ b/internal/pool/index/seed_test.go @@ -0,0 +1,39 @@ +package index + +import "testing" + +// The seed this build carries is a document a reader accepts: it parses, +// declares the pool's quality metric as gaussian, and carries at least one +// measurement for each of the three seats the picker resolves — so a machine +// with no cache has numbers on its first run rather than an empty index. +func TestSeedCarriesRoleQualityForEverySeat(t *testing.T) { + seed, err := SeedIndex() + if err != nil { + t.Fatalf("the embedded seed does not parse: %v", err) + } + if kind, ok := seed.Kind("role_quality"); !ok || kind != "gaussian" { + t.Fatalf("role_quality: kind %q, declared %v", kind, ok) + } + byRole := map[string]int{} + for _, cell := range seed.Cells("role_quality") { + byRole[cell.Role]++ + } + for _, role := range []string{"worker", "high", "mastermind"} { + if byRole[role] == 0 { + t.Errorf("the seed carries no role_quality cell for %q", role) + } + } +} + +// Seed hands back a copy: the document is shared by many readers, and one that +// mutated the slice it was given would change what every later call returns. +func TestSeedHandsBackACopy(t *testing.T) { + first := Seed() + if len(first) == 0 { + t.Fatal("the seed is empty") + } + first[0] ^= 0xff + if second := Seed(); second[0] == first[0] { + t.Fatal("mutating a Seed copy changed the next one") + } +} diff --git a/internal/pool/judge/judge.go b/internal/pool/judge/judge.go new file mode 100644 index 0000000000..95c4cf6626 --- /dev/null +++ b/internal/pool/judge/judge.go @@ -0,0 +1,344 @@ +// Package judge scores a finished run seat by seat, independently of the crew +// that produced it. +// +// THE MECHANISM. A run's record names which model held each judged seat — +// worker, high, mastermind — and what the run delivered. Every seat the record +// carries is put to one model that was not in the crew: the caller supplies +// [Ask], which sends a system and a user prompt and returns the answer. One +// question is asked per seat, in role order, and each answer is read as one +// JSON object carrying a score from 0 to 100 and a one-sentence reason. That is +// the same scale crewpick reads seat quality on and a pool records its +// role_quality metric in, so a run's seats land on the scale the picker already +// uses. +// +// THE CALLER OWNS THE CALL. This package touches no disk and no network and +// holds no credentials: the model, the transport and the bill are the caller's, +// and the caller bills the judge's calls to its own seat. [Pick] chooses that +// model from a catalog, and the chosen id rides in the record ([Record.Judge]) +// so a seat already holding it is skipped rather than scoring its own work. +package judge + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "sort" + "strconv" + "strings" + + "github.com/Agent-Field/codeaf/internal/catalog" +) + +// recordClip is how many runes of the brief and of the deliverable the user +// prompt carries. They are the two fields that grow with the size of the work +// rather than with its shape, and a judge scoring a seat does not need the whole +// of either to say whether the seat did its job. The report, the claim and the +// ending travel whole: they are the run's own account of itself, and cutting +// them would score a seat on a sentence it did not finish. +const recordClip = 6000 + +// DefaultFloor is the published coding index a judge must reach to be picked. +// A judge below the worker's own class rubber-stamps the work rather than +// scoring it: it has less of the capability the seat is scored on than the seat +// it is reading. +const DefaultFloor = 60 + +// Role is one of a crew's judged seats. +type Role string + +const ( + // RoleWorker is the seat that carried the work. + RoleWorker Role = "worker" + // RoleHigh is the seat that checked the work. + RoleHigh Role = "high" + // RoleMastermind is the seat that planned the work. + RoleMastermind Role = "mastermind" +) + +// seatOrder is the order seats are asked in: the work, then what checked it, +// then the plan it was cut under. +var seatOrder = []Role{RoleWorker, RoleHigh, RoleMastermind} + +// Record is one finished run as the judge reads it: the spec it ran under, its +// own account of itself, what it touched, and who held each seat. It is the +// judge's own copy, assembled by the caller from wherever the run is recorded, +// so this package imports neither the session store nor anything else that +// owns the run. +type Record struct { + // Brief is what the run was asked to do. + Brief string + // Deliverable is what had to exist when it was over. + Deliverable string + // Report is the account the run landed with. + Report string + // Claim is the run's own summary of its answer, kept beside the report. + Claim string + // Ending is why a run that stopped early stopped where it did. + Ending string + // Files is every path the run wrote. + Files []string + // Changed is how many paths it changed. + Changed int + // Checks is the repeatable verification the run ran. + Checks []string + // Seats is which model held each judged seat. The chat door asks no planner + // call, so a record may carry only the worker and the high seats. + Seats map[Role]string + // Judge is the model that will answer the seat questions, the id [Pick] + // returned. A seat already holding it is skipped, so a model never scores + // its own work; an empty Judge names no model and skips nothing. + Judge string + // CostUSD is what the run spent, in dollars. + CostUSD float64 +} + +// Score is one seat's reading: the role, the model that held it, the score on +// the 0-100 scale and the judge's one-sentence reason. +type Score struct { + Role Role + Model string + Score float64 + Reason string +} + +// Ask puts one question to the judge model and returns its answer. The caller +// owns the model, the credentials and the transport, and bills the call to the +// judge's own seat; this package writes both prompts and reads the answer. +type Ask func(ctx context.Context, system, user string) (answer string, err error) + +// Judge asks one question per seat present in rec.Seats, in role order, and +// answers with a Score for each seat that was read. A seat held by the judge's +// own model ([Record.Judge]) is skipped and named in the returned error; a seat +// whose call fails, or whose answer is not one JSON object carrying an integer +// score from 0 to 100, is an error for that seat alone and never stops the +// others. The scores obtained are returned beside a joined error naming every +// seat that failed, and a record with no seats returns no scores and no error. +func Judge(ctx context.Context, ask Ask, rec Record) ([]Score, error) { + if len(rec.Seats) == 0 { + return nil, nil + } + if ask == nil { + return nil, errors.New("judge: no ask function") + } + scores := make([]Score, 0, len(seatOrder)) + var failures []error + for _, role := range seatOrder { + model, present := rec.Seats[role] + if !present { + continue + } + if rec.Judge != "" && model == rec.Judge { + failures = append(failures, fmt.Errorf("judge: the %s seat is held by the judge's own model and was skipped", role)) + continue + } + answer, err := ask(ctx, seatSystem(role), seatUser(role, model, rec)) + if err != nil { + failures = append(failures, fmt.Errorf("judge: the %s seat: %w", role, err)) + continue + } + score, reason, err := readAnswer(answer) + if err != nil { + failures = append(failures, fmt.Errorf("judge: the %s seat: %w", role, err)) + continue + } + scores = append(scores, Score{Role: role, Model: model, Score: score, Reason: reason}) + } + if len(failures) > 0 { + return scores, errors.Join(failures...) + } + return scores, nil +} + +// Candidates returns every catalog row that can judge a crew, cheapest first. +// +// A row qualifies exactly as the judge a crew would otherwise be handed does: +// its published coding index reaches floor, it carries "tools", no crew seat +// holds it, and its vendor differs from the worker's. Rows the provider +// published no price for are out of the running. TWO MORE ROWS ARE NEVER +// CANDIDATES: one whose prompt and completion prices sum to zero, and one +// whose id ends in ":free" whatever the case. A free row is a rate-limited +// row and not a price — the provider answers it with a 429 at the moment a +// judge most needs an answer — so it is asked for nothing, whatever its index +// says. +// +// The list is ordered by cost and then by id, so the cheapest stands first and +// the answer is a property of the catalog rather than of its order. A crew +// with nothing left to draw from yields no candidates. +func Candidates(models []catalog.Model, crew map[Role]string, floor float64) []string { + workerVendor := "" + if worker, held := crew[RoleWorker]; held && worker != "" { + workerVendor = vendor(worker) + } + held := make(map[string]bool, len(crew)) + for _, model := range crew { + if model != "" { + held[model] = true + } + } + type candidate struct { + id string + cost float64 + } + var found []candidate + for _, model := range models { + if model.PriceUnknown || model.CodingIndex < floor || !hasTools(model.Parameters) || held[model.ID] { + continue + } + if workerVendor != "" && vendor(model.ID) == workerVendor { + continue + } + if model.PromptPrice+model.CompletionPrice == 0 { + continue + } + if strings.HasSuffix(strings.ToLower(model.ID), ":free") { + continue + } + found = append(found, candidate{model.ID, model.PromptPrice + model.CompletionPrice}) + } + sort.Slice(found, func(i, j int) bool { + if found[i].cost != found[j].cost { + return found[i].cost < found[j].cost + } + return found[i].id < found[j].id + }) + ids := make([]string, 0, len(found)) + for _, c := range found { + ids = append(ids, c.id) + } + return ids +} + +// Pick returns the cheapest judge [Candidates] names, and ok is false when it +// names none. It is Candidates' head-or-nothing, so the properties its callers +// rely on hold here too: the cheaper of two rows wins, and a tie breaks to the +// lower id. +func Pick(models []catalog.Model, crew map[Role]string, floor float64) (judgeID string, ok bool) { + candidates := Candidates(models, crew, floor) + if len(candidates) == 0 { + return "", false + } + return candidates[0], true +} + +// seatJobs is what each seat is responsible for, and therefore what its work is +// scored against. One paragraph per seat, read as the opening of the system +// prompt. +var seatJobs = map[Role]string{ + RoleWorker: "You are scoring the WORKER seat of a finished run. The worker seat carried the work: judge whether the delivered work did what the brief asked, coherently, and with the checks the record says were run.", + RoleHigh: "You are scoring the HIGH seat of a finished run. The high seat checked the work: judge whether the check read the work itself rather than the worker's claim about it, and whether it caught what a careful reader of the work would catch.", + RoleMastermind: "You are scoring the MASTERMIND seat of a finished run. The mastermind seat planned the work: judge whether the work was cut into steps the brief supports.", +} + +// answerShape is the one object every seat's answer is read as, and nothing +// else. It is stated in both prompts because a model that answers with prose +// around the object is still read, but a model that answers with no object at +// all fails its seat. +const answerShape = `Answer with ONE JSON object and nothing else: {"score": <0-100 integer>, "reason": ""}` + +// seatSystem is the system prompt for one seat: the seat's job followed by the +// shape of the one answer asked for. +func seatSystem(role Role) string { + return seatJobs[role] + " " + answerShape +} + +// seatUser is the user prompt for one seat: the record's fields, the brief and +// the deliverable clipped to recordClip runes each, and a line naming which +// model held the seat. The model id is named and nothing is said about its +// maker — the seat is scored on the work, not on the name behind it. +func seatUser(role Role, model string, rec Record) string { + var b strings.Builder + fmt.Fprintf(&b, "Seat: %s\n", role) + fmt.Fprintf(&b, "Model holding this seat: %s\n\n", model) + fmt.Fprintf(&b, "Brief:\n%s\n\n", clip(rec.Brief)) + fmt.Fprintf(&b, "Deliverable:\n%s\n\n", clip(rec.Deliverable)) + fmt.Fprintf(&b, "Report:\n%s\n\n", rec.Report) + fmt.Fprintf(&b, "Claim:\n%s\n\n", rec.Claim) + fmt.Fprintf(&b, "Ending:\n%s\n\n", rec.Ending) + fmt.Fprintf(&b, "Checks:\n%s\n\n", strings.Join(rec.Checks, "\n")) + fmt.Fprintf(&b, "Files:\n%s\n\n", strings.Join(rec.Files, "\n")) + fmt.Fprintf(&b, "Changed: %d\n", rec.Changed) + fmt.Fprintf(&b, "Cost (USD): %g\n", rec.CostUSD) + return b.String() +} + +// clip returns s cut to at most recordClip runes, so neither the brief nor the +// deliverable can push the prompt past the budget whatever the run was. +func clip(s string) string { + if len(s) <= recordClip { + return s + } + runes := []rune(s) + if len(runes) <= recordClip { + return s + } + return string(runes[:recordClip]) +} + +// readAnswer reads the first JSON object in the answer into a score and a +// reason. A score that is missing, not a number, not an integer, or outside 0 +// to 100 is an error, and so is an answer carrying no JSON object at all. The +// reason may be absent and reads as empty. +func readAnswer(answer string) (float64, string, error) { + raw, found := firstObject(answer) + if !found { + return 0, "", errors.New("the answer carries no JSON object") + } + var got struct { + Score *json.Number `json:"score"` + Reason string `json:"reason"` + } + if err := json.Unmarshal(raw, &got); err != nil { + return 0, "", fmt.Errorf("the answer is not a JSON object: %w", err) + } + if got.Score == nil { + return 0, "", errors.New("the answer carries no score") + } + score, err := strconv.ParseFloat(got.Score.String(), 64) + if err != nil || math.IsNaN(score) || math.IsInf(score, 0) { + return 0, "", fmt.Errorf("the score %q is not a number", got.Score.String()) + } + if score != math.Trunc(score) { + return 0, "", fmt.Errorf("the score %v is not an integer", score) + } + if score < 0 || score > 100 { + return 0, "", fmt.Errorf("the score %v is outside 0-100", score) + } + return score, got.Reason, nil +} + +// firstObject reads the first JSON value that opens at the answer's first "{", +// so prose before the object costs nothing and prose after it is ignored. An +// answer with no "{" and an object that does not parse both answer with false. +func firstObject(answer string) (json.RawMessage, bool) { + open := strings.IndexByte(answer, '{') + if open < 0 { + return nil, false + } + var raw json.RawMessage + if err := json.NewDecoder(strings.NewReader(answer[open:])).Decode(&raw); err != nil { + return nil, false + } + return raw, true +} + +// hasTools reports whether the provider published "tools" among the fields a +// model accepts. +func hasTools(parameters []string) bool { + for _, parameter := range parameters { + if strings.EqualFold(strings.TrimSpace(parameter), "tools") { + return true + } + } + return false +} + +// vendor is the part of a model id before the first slash — "z-ai" in +// "z-ai/glm-5.3" — and the whole id when there is no slash. +func vendor(id string) string { + if slash := strings.IndexByte(id, '/'); slash > 0 { + return id[:slash] + } + return id +} diff --git a/internal/pool/judge/judge_test.go b/internal/pool/judge/judge_test.go new file mode 100644 index 0000000000..c4b8687403 --- /dev/null +++ b/internal/pool/judge/judge_test.go @@ -0,0 +1,307 @@ +package judge + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/catalog" +) + +// fakeAsk records every prompt it is handed and answers with the replies it was +// given, in order, or with an error when one was set. +type fakeAsk struct { + systems []string + users []string + replies []string + err error +} + +func (f *fakeAsk) ask(_ context.Context, system, user string) (string, error) { + f.systems = append(f.systems, system) + f.users = append(f.users, user) + if f.err != nil { + return "", f.err + } + if len(f.replies) == 0 { + return `{"score": 1, "reason": "no reply was set"}`, nil + } + reply := f.replies[0] + f.replies = f.replies[1:] + return reply, nil +} + +func TestAScoreComesBackForEverySeatInRoleOrder(t *testing.T) { + rec := Record{ + Brief: "do the thing", + Seats: map[Role]string{RoleMastermind: "v/mind", RoleHigh: "v/high", RoleWorker: "v/worker"}, + } + ask := &fakeAsk{replies: []string{ + `{"score": 80, "reason": "the worker did what the brief asked"}`, + `{"score": 70, "reason": "the check read the work"}`, + `{"score": 60, "reason": "the plan held the brief"}`, + }} + scores, err := Judge(context.Background(), ask.ask, rec) + if err != nil { + t.Fatalf("judge: %v", err) + } + want := []struct { + role Role + model string + score float64 + }{ + {RoleWorker, "v/worker", 80}, + {RoleHigh, "v/high", 70}, + {RoleMastermind, "v/mind", 60}, + } + if len(scores) != len(want) { + t.Fatalf("got %d scores, want %d", len(scores), len(want)) + } + for i, w := range want { + if scores[i].Role != w.role || scores[i].Model != w.model || scores[i].Score != w.score { + t.Errorf("score %d is %+v, want %s %s %v", i, scores[i], w.role, w.model, w.score) + } + if scores[i].Reason == "" { + t.Errorf("score %d carries no reason", i) + } + } +} + +func TestASeatTheJudgeHoldsIsSkippedAndNamed(t *testing.T) { + rec := Record{ + Seats: map[Role]string{RoleWorker: "v/worker", RoleHigh: "v/judge", RoleMastermind: "v/mind"}, + Judge: "v/judge", + } + ask := &fakeAsk{replies: []string{ + `{"score": 80, "reason": "ok"}`, + `{"score": 60, "reason": "ok"}`, + }} + scores, err := Judge(context.Background(), ask.ask, rec) + if err == nil { + t.Fatal("expected an error naming the skipped seat") + } + if !strings.Contains(err.Error(), "high") { + t.Errorf("the error %q does not name the high seat", err) + } + if len(scores) != 2 { + t.Fatalf("got %d scores, want 2", len(scores)) + } + for _, score := range scores { + if score.Role == RoleHigh { + t.Error("the high seat was scored though the judge holds it") + } + } + if len(ask.systems) != 2 { + t.Errorf("the judge was asked %d questions, want 2", len(ask.systems)) + } +} + +func TestAMalformedAnswerFailsOnlyItsSeat(t *testing.T) { + rec := Record{Seats: map[Role]string{RoleWorker: "v/worker", RoleHigh: "v/high"}} + ask := &fakeAsk{replies: []string{"I would rather not say.", `{"score": 70, "reason": "fine"}`}} + scores, err := Judge(context.Background(), ask.ask, rec) + if err == nil { + t.Fatal("expected an error naming the worker seat") + } + if !strings.Contains(err.Error(), "worker") { + t.Errorf("the error %q does not name the worker seat", err) + } + if len(scores) != 1 || scores[0].Role != RoleHigh { + t.Fatalf("got %+v, want only the high seat scored", scores) + } +} + +func TestAScoreOutsideTheScaleFailsItsSeat(t *testing.T) { + rec := Record{Seats: map[Role]string{RoleWorker: "v/worker", RoleHigh: "v/high"}} + ask := &fakeAsk{replies: []string{`{"score": 140, "reason": "off the scale"}`, `{"score": 70, "reason": "fine"}`}} + scores, err := Judge(context.Background(), ask.ask, rec) + if err == nil { + t.Fatal("expected an error for the out-of-range score") + } + if len(scores) != 1 || scores[0].Role != RoleHigh { + t.Fatalf("got %+v, want only the high seat scored", scores) + } +} + +func TestPromptsStayInsideTheClipBudget(t *testing.T) { + full := strings.Repeat("b", recordClip) + rec := Record{ + Brief: full + "BRIEF-TAIL", + Deliverable: full + "DELIVERABLE-TAIL", + Report: strings.Repeat("r", recordClip+500) + "REPORT-TAIL", + Seats: map[Role]string{RoleWorker: "v/worker"}, + } + ask := &fakeAsk{replies: []string{`{"score": 50, "reason": "ok"}`}} + if _, err := Judge(context.Background(), ask.ask, rec); err != nil { + t.Fatalf("judge: %v", err) + } + user := ask.users[0] + if strings.Contains(user, "BRIEF-TAIL") { + t.Error("the brief was carried past the clip budget") + } + if strings.Contains(user, "DELIVERABLE-TAIL") { + t.Error("the deliverable was carried past the clip budget") + } + if !strings.Contains(user, "REPORT-TAIL") { + t.Error("the report was clipped though it travels whole") + } +} + +func TestARecordWithNoSeatsScoresNothingAndFailsNothing(t *testing.T) { + ask := &fakeAsk{err: errors.New("the judge should not have been asked")} + scores, err := Judge(context.Background(), ask.ask, Record{Brief: "nothing ran"}) + if err != nil { + t.Fatalf("judge: %v", err) + } + if len(scores) != 0 { + t.Errorf("got %d scores, want none", len(scores)) + } + if len(ask.systems) != 0 { + t.Errorf("the judge was asked %d questions with no seats to read", len(ask.systems)) + } +} + +func TestPickExcludesCrewModelsAndTheWorkerVendor(t *testing.T) { + models := []catalog.Model{ + {ID: "worker-vendor/cheap", PromptPrice: 0.1, CompletionPrice: 0.1, CodingIndex: 90, Parameters: []string{"tools"}}, + {ID: "other/crew-held", PromptPrice: 0.2, CompletionPrice: 0.2, CodingIndex: 90, Parameters: []string{"tools"}}, + {ID: "judge-vendor/fine", PromptPrice: 1, CompletionPrice: 1, CodingIndex: 80, Parameters: []string{"tools"}}, + } + crew := map[Role]string{RoleWorker: "worker-vendor/w", RoleHigh: "other/crew-held"} + got, ok := Pick(models, crew, DefaultFloor) + if !ok { + t.Fatal("Pick found no judge though one qualifies") + } + if got != "judge-vendor/fine" { + t.Errorf("the judge is %q, want judge-vendor/fine", got) + } +} + +func TestPickHonoursTheFloorAndSkipsUnpricedRows(t *testing.T) { + models := []catalog.Model{ + {ID: "a/below-floor", PromptPrice: 0.01, CompletionPrice: 0.01, CodingIndex: 40, Parameters: []string{"tools"}}, + {ID: "b/unpriced", PromptPrice: 0, CompletionPrice: 0, PriceUnknown: true, CodingIndex: 99, Parameters: []string{"tools"}}, + {ID: "c/no-tools", PromptPrice: 0.02, CompletionPrice: 0.02, CodingIndex: 90}, + {ID: "d/qualifies", PromptPrice: 0.5, CompletionPrice: 0.5, CodingIndex: 80, Parameters: []string{"tools"}}, + } + got, ok := Pick(models, nil, DefaultFloor) + if !ok || got != "d/qualifies" { + t.Errorf("the judge is %q, ok %v, want d/qualifies", got, ok) + } + if _, ok := Pick(nil, nil, DefaultFloor); ok { + t.Error("Pick found a judge in an empty catalog") + } +} + +func TestPickIsDeterministicUnderShuffledInput(t *testing.T) { + first := catalog.Model{ID: "a/one", PromptPrice: 1, CompletionPrice: 1, CodingIndex: 80, Parameters: []string{"tools"}} + second := catalog.Model{ID: "b/two", PromptPrice: 1, CompletionPrice: 1, CodingIndex: 80, Parameters: []string{"tools"}} + forward, _ := Pick([]catalog.Model{first, second}, nil, DefaultFloor) + backward, _ := Pick([]catalog.Model{second, first}, nil, DefaultFloor) + if forward != backward { + t.Errorf("Pick depends on order: %q then %q", forward, backward) + } + if forward != "a/one" { + t.Errorf("a tie picked %q, want the lower id a/one", forward) + } +} + +func TestCandidatesNeverNamesAFreeRowWhileAPaidRowQualifies(t *testing.T) { + models := []catalog.Model{ + {ID: "free/vendor:free", PromptPrice: 0, CompletionPrice: 0, CodingIndex: 90, Parameters: []string{"tools"}}, + {ID: "zero-priced/vendor", PromptPrice: 0, CompletionPrice: 0, CodingIndex: 80, Parameters: []string{"tools"}}, + {ID: "paid/vendor", PromptPrice: 0.5, CompletionPrice: 0.5, CodingIndex: 65, Parameters: []string{"tools"}}, + } + got := Candidates(models, nil, DefaultFloor) + if len(got) != 1 || got[0] != "paid/vendor" { + t.Fatalf("Candidates named %v, want only the paid row", got) + } +} + +func TestCandidatesIsOrderedByCostThenIdAndExcludesWhatPickExcludes(t *testing.T) { + models := []catalog.Model{ + {ID: "b/dear", PromptPrice: 1, CompletionPrice: 1, CodingIndex: 80, Parameters: []string{"tools"}}, + {ID: "crew-held/cheap", PromptPrice: 0.1, CompletionPrice: 0.1, CodingIndex: 99, Parameters: []string{"tools"}}, + {ID: "worker-vendor/cheap", PromptPrice: 0.1, CompletionPrice: 0.1, CodingIndex: 90, Parameters: []string{"tools"}}, + {ID: "a/cheap", PromptPrice: 0.5, CompletionPrice: 0.5, CodingIndex: 80, Parameters: []string{"tools"}}, + {ID: "c/tie", PromptPrice: 0.5, CompletionPrice: 0.5, CodingIndex: 80, Parameters: []string{"tools"}}, + {ID: "d/below-floor", PromptPrice: 0.01, CompletionPrice: 0.01, CodingIndex: 40, Parameters: []string{"tools"}}, + } + crew := map[Role]string{RoleWorker: "worker-vendor/w", RoleHigh: "crew-held/cheap"} + got := Candidates(models, crew, DefaultFloor) + want := []string{"a/cheap", "c/tie", "b/dear"} + if len(got) != len(want) { + t.Fatalf("Candidates named %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("Candidates named %v, want %v", got, want) + } + } +} + +// roundTripFunc is the fake transport the snapshot catalog is read through: it +// answers every request with the captured bytes, the way internal/catalog's own +// captured-response tests do. +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return fn(request) } + +// snapshotCatalog loads the captured OpenRouter listing internal/catalog's own +// tests read, through a transport that answers with the bytes on disk. +func snapshotCatalog(t *testing.T) *catalog.Catalog { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "catalog", "testdata", "openrouter-models.json")) + if err != nil { + t.Fatal(err) + } + client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Status: http.StatusText(http.StatusOK), + Header: make(http.Header), + Body: io.NopCloser(bytes.NewReader(raw)), + Request: request, + }, nil + })} + return catalog.Load(context.Background(), catalog.Options{ + BaseURL: catalog.DefaultBaseURL, + Dir: t.TempDir(), + HTTPClient: client, + }) +} + +func TestCandidatesFromTheCapturedListingAreAllPaidAndNoneFree(t *testing.T) { + c := snapshotCatalog(t) + models := c.ModelsNow() + if len(models) == 0 { + t.Fatal("the captured listing read back no rows") + } + crew := map[Role]string{RoleWorker: "z-ai/glm-5.3-flash", RoleHigh: "anthropic/claude-fable-5.1"} + got := Candidates(models, crew, DefaultFloor) + if len(got) == 0 { + t.Fatal("the captured listing yielded no judge candidates") + } + byID := make(map[string]catalog.Model, len(models)) + for _, model := range models { + byID[model.ID] = model + } + for _, id := range got { + if strings.HasSuffix(strings.ToLower(id), ":free") { + t.Errorf("the candidate %q is a free row", id) + } + model, ok := byID[id] + if !ok { + t.Fatalf("the candidate %q is not a row of the listing", id) + } + if model.PromptPrice+model.CompletionPrice <= 0 { + t.Errorf("the candidate %q carries no price", id) + } + } +} diff --git a/internal/pool/outbox/outbox.go b/internal/pool/outbox/outbox.go new file mode 100644 index 0000000000..919b05191a --- /dev/null +++ b/internal/pool/outbox/outbox.go @@ -0,0 +1,554 @@ +// Package outbox keeps small measurement rows in a local append-only file and +// hands them to a destination: an HTTP endpoint that takes newline-delimited +// JSON, or another file. One line is one row, so the rows survive a restart +// and Pending() is answered by reading the file back. Nothing here gets in the +// caller's way — a full disk, a dead server, a destination that is nonsense +// all surface as error values — and a row whose batch arrived is marked in the +// file, so it is never transmitted twice. +// +// The file holds rows and, interleaved with them, one-line sent and dropped +// markers that retire rows; a line that is not a row is skipped on read. That +// is what keeps the file append-only while a cap on pending rows still drops +// from the old end and a mark outlives closing the outbox and opening the same +// path again. +package outbox + +import ( + "bytes" + "context" + cryptorand "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// DefaultMaxPending is the cap on pending rows when Outbox.MaxPending is zero +// or below. +const DefaultMaxPending = 5000 + +const ( + rowSchema = 1 + nonceLen = 16 + maxPayload = 64 << 10 // 64 KiB, measured on the compacted payload + maxBatch = 200 // rows per POST + // defaultBudget bounds one Send when Outbox.Budget is zero. + defaultBudget = 2 * time.Second + dirMode = 0700 + fileMode = 0600 +) + +// Row is one measurement as it is stored in the outbox file, one row per line. +type Row struct { + Schema int `json:"schema"` // always 1 + Day string `json:"day"` // "YYYY-MM-DD", UTC + Nonce string `json:"nonce"` // 16 random bytes, lowercase hex + Payload json.RawMessage `json:"payload"` +} + +// mark retires rows: a retired row is no longer pending, either because its +// batch arrived or because the pending cap dropped it. A marker is a line of +// its own in the same file, which is what keeps the outbox append-only. +type mark struct { + Sent string `json:"sent,omitempty"` + Dropped string `json:"dropped,omitempty"` +} + +// Outbox keeps rows in one append-only file until they are sent. The six +// exported fields are read every time they are used, so a caller may set them +// after Open and before the first Append or Send. +type Outbox struct { + // Now answers the time a row was appended; nil means time.Now. + Now func() time.Time + // Rand is read for nonce bytes; nil means crypto/rand.Reader. + Rand io.Reader + // Keep decides, at Send time, which payloads are transmitted; nil keeps + // every row. A payload Keep declines leaves the outbox without reaching + // the destination. + Keep func(payload json.RawMessage) bool + // Budget bounds one Send call as a whole; zero means 2s. + Budget time.Duration + // Client sends the batches; nil means a client the package makes. + Client *http.Client + // MaxPending caps the pending rows, dropping from the old end; zero or + // below means DefaultMaxPending. + MaxPending int + // Install is the install's own nonce, carried as the X-Codeaf-Install + // header on every batch sent over http; empty sends no header. A + // destination that requires the header names it in its error, so a batch + // sent without one is a batch the destination refuses — the caller sets + // this from its own store before the first Send. + Install string + + path string + sendMu sync.Mutex // one Send at a time, so two never carry the same rows + mu sync.Mutex // guards f and pend + f *os.File + pend []string // nonces of the pending rows, oldest first +} + +// defaultClient is shared by every outbox that does not name its own. Its +// requests are bounded by the Send budget, so no client-level timeout. +var defaultClient = &http.Client{} + +// Open prepares the outbox file at path, creating the parent directory with +// mode 0700 if it is missing and the file itself with mode 0600. A path that +// already holds rows keeps them, markers included. An empty path is an error. +func Open(path string) (*Outbox, error) { + if path == "" { + return nil, errors.New("outbox: empty path") + } + if err := os.MkdirAll(filepath.Dir(path), dirMode); err != nil { + return nil, err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, fileMode) + if err != nil { + return nil, err + } + o := &Outbox{path: path, f: f} + o.mu.Lock() + defer o.mu.Unlock() + rows := o.load() + o.pend = make([]string, 0, len(rows)) + for _, r := range rows { + o.pend = append(o.pend, r.Nonce) + } + return o, nil +} + +// Append writes one row holding payload. The row is refused, and nothing is +// written, when the payload is not valid JSON, when it is longer than 64 KiB +// once compacted, or when it still holds a newline once compacted. The payload +// is stored compacted, so one row is always one line. +func (o *Outbox) Append(payload json.RawMessage) error { + compacted, err := compact(payload) + if err != nil { + return err + } + if len(compacted) > maxPayload { + return errors.New("outbox: payload is longer than 64 KiB once compacted") + } + if bytes.IndexByte(compacted, '\n') >= 0 { + return errors.New("outbox: payload holds a newline once compacted") + } + nonce := make([]byte, nonceLen) + rand := o.Rand + if rand == nil { + rand = cryptorand.Reader + } + if _, err := io.ReadFull(rand, nonce); err != nil { + return fmt.Errorf("outbox: read nonce: %w", err) + } + now := o.Now + var t time.Time + if now == nil { + t = time.Now() + } else { + t = now() + } + row := Row{ + Schema: rowSchema, + Day: t.UTC().Format("2006-01-02"), + Nonce: hex.EncodeToString(nonce), + Payload: json.RawMessage(compacted), + } + line, err := rowLine(row) + if err != nil { + return err + } + + o.mu.Lock() + defer o.mu.Unlock() + if o.f == nil { + return errors.New("outbox: outbox is closed") + } + if _, err := o.f.Write(line); err != nil { + return err + } + o.pend = append(o.pend, row.Nonce) + o.trimLocked() + return nil +} + +// Pending answers the rows that have not been sent, in the order they were +// appended. It never fails: a line that does not parse, or that is not a row, +// is skipped, and a file that cannot be read answers as no rows at all. +func (o *Outbox) Pending() []Row { + o.mu.Lock() + defer o.mu.Unlock() + return o.load() +} + +// Send hands the pending rows to dest and answers with how many rows it +// transmitted. A dest beginning https:// or http:// receives one POST per +// batch of at most 200 rows, content-type application/x-ndjson, the body one +// row per line in pending order, each line the same JSON object the file +// holds; a 2xx answer means that batch arrived. Any other dest is a +// filesystem path, plain or written file://, that the rows are appended to, +// created with mode 0600 and a parent directory of mode 0700 if either is +// missing. An empty dest, or a Send with nothing pending, is a no-op: 0 and a +// nil error, the rows staying pending. +// +// The whole call lives inside Budget, which is 2s when it is zero, and inside +// ctx as well, whichever ends first. Rows whose batch arrived are marked sent +// and leave the outbox; a row Keep declines is marked sent without being +// transmitted and is not counted. Rows whose batch did not arrive, and every +// row after it, stay pending, and Send returns an error that says what +// failed. +func (o *Outbox) Send(ctx context.Context, dest string) (int, error) { + if dest == "" { + return 0, nil + } + o.sendMu.Lock() + defer o.sendMu.Unlock() + + o.mu.Lock() + if o.f == nil { + o.mu.Unlock() + return 0, errors.New("outbox: outbox is closed") + } + rows := o.load() + pend := make([]string, len(rows)) + for i, r := range rows { + pend[i] = r.Nonce + } + o.pend = pend + o.mu.Unlock() + + if len(rows) == 0 { + return 0, nil + } + + budget := o.Budget + if budget <= 0 { + budget = defaultBudget + } + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(budget)) + defer cancel() + + // Rows the caller declines leave the outbox at once, whether or not any + // later batch arrives. + keep := o.Keep + var deliver []Row + if keep == nil { + deliver = rows + } else { + deliver = make([]Row, 0, len(rows)) + var declined []string + for _, r := range rows { + if keep(r.Payload) { + deliver = append(deliver, r) + } else { + declined = append(declined, r.Nonce) + } + } + if len(declined) > 0 { + o.retire(sentMarks(declined)) + } + } + if len(deliver) == 0 { + return 0, nil + } + + if strings.HasPrefix(dest, "https://") || strings.HasPrefix(dest, "http://") { + return o.sendHTTP(ctx, dest, deliver) + } + return o.sendFile(ctx, strings.TrimPrefix(dest, "file://"), deliver) +} + +// Close releases whatever the outbox holds open. Calling it more than once is +// not an error. A closed outbox refuses Append and Send; Pending still +// answers by reading the file. +func (o *Outbox) Close() error { + o.mu.Lock() + defer o.mu.Unlock() + if o.f == nil { + return nil + } + err := o.f.Close() + o.f = nil + return err +} + +// trimLocked drops the oldest pending rows until the cap holds, writing a +// dropped marker for each so they stay gone across a reopen. The row just +// appended is the last one, so it always survives. The caller holds mu. +func (o *Outbox) trimLocked() { + limit := o.MaxPending + if limit <= 0 { + limit = DefaultMaxPending + } + drop := len(o.pend) - limit + if drop <= 0 { + return + } + dropped := make([]string, drop) + copy(dropped, o.pend[:drop]) + if o.writeMarksLocked(droppedMarks(dropped)) != nil { + // The cap is kept by the markers on disk, so a cap this Append could + // not record is a cap that is not kept. The rows stay pending and the + // next Append tries the trim again, which is the same answer the + // failed row write itself gives. + return + } + o.pend = append(o.pend[:0], o.pend[drop:]...) +} + +// retire appends one marker per nonce and takes those rows out of the pending +// set. A marker that could not be written leaves its rows pending, where the +// next Send tries them again rather than losing them — which is why the drop +// happens only once the write has gone through. The alternative, a row gone +// from the pending set and present in the file, is a row that comes back at +// the next Open having been reported delivered. +func (o *Outbox) retire(marks []mark) { + o.mu.Lock() + defer o.mu.Unlock() + if o.writeMarksLocked(marks) != nil { + return + } + o.dropPendingLocked(marks) +} + +// writeMarksLocked appends one marker line per mark, and answers whether the +// line reached the file. The caller holds mu. +func (o *Outbox) writeMarksLocked(marks []mark) error { + if len(marks) == 0 { + return nil + } + if o.f == nil { + return errors.New("outbox: outbox is closed") + } + buf := make([]byte, 0, 24*len(marks)) + for _, m := range marks { + line, err := json.Marshal(m) + if err != nil { + continue // a struct of strings never fails to marshal + } + buf = append(buf, line...) + buf = append(buf, '\n') + } + _, err := o.f.Write(buf) + return err +} + +// dropPendingLocked removes the marked rows from the pending list, ONE ROW PER +// MARKER and oldest first. A nonce is not an identity: [Outbox.Rand] is the +// caller's to set, and a source that answers with the same bytes twice writes +// the same nonce twice. Treating the marks as a set of names would then take +// every row sharing a name — the rows a failed batch left pending, the row a +// trim was told to keep — out of the outbox with it. Markers are only ever +// written for the oldest rows, so counting them off the front is what they +// mean. The caller holds mu. +func (o *Outbox) dropPendingLocked(marks []mark) { + gone := make(map[string]int, len(marks)) + for _, m := range marks { + if m.Sent != "" { + gone[m.Sent]++ + } + if m.Dropped != "" { + gone[m.Dropped]++ + } + } + kept := o.pend[:0] + for _, n := range o.pend { + if gone[n] > 0 { + gone[n]-- + continue + } + kept = append(kept, n) + } + o.pend = kept +} + +// load reads the outbox file and answers the rows still pending, in file +// order. A line that does not parse, or that is not a row, is skipped, and a +// file that cannot be read answers as no rows at all. The caller holds mu. +// +// EACH MARKER RETIRES ONE ROW, oldest first, wherever the marker sits — see +// [Outbox.dropPendingLocked] for why the count rather than the name: a nonce +// is what the caller's own [Outbox.Rand] made of sixteen bytes, and a reader +// that took a marker as a name would answer that a file holding two rows under +// one nonce and one marker holds nothing at all. +func (o *Outbox) load() []Row { + b, err := os.ReadFile(o.path) + if err != nil { + return nil + } + var rows []Row + retired := make(map[string]int) + for len(b) > 0 { + line := b + if i := bytes.IndexByte(b, '\n'); i >= 0 { + line, b = b[:i], b[i+1:] + } else { + b = nil + } + if len(line) == 0 { + continue + } + var m mark + if json.Unmarshal(line, &m) == nil { + if m.Sent != "" { + retired[m.Sent]++ + continue + } + if m.Dropped != "" { + retired[m.Dropped]++ + continue + } + } + var r Row + if json.Unmarshal(line, &r) != nil || r.Nonce == "" { + continue + } + rows = append(rows, r) + } + kept := rows[:0] + for _, r := range rows { + if retired[r.Nonce] > 0 { + retired[r.Nonce]-- + continue + } + kept = append(kept, r) + } + return kept +} + +// sendHTTP posts the rows to dest, at most maxBatch per POST, and marks each +// batch that came back 2xx. The first batch that fails ends the call; its +// rows and every row after it stay pending. +func (o *Outbox) sendHTTP(ctx context.Context, dest string, rows []Row) (int, error) { + client := o.Client + if client == nil { + client = defaultClient + } + n := 0 + for start := 0; start < len(rows); start += maxBatch { + end := start + maxBatch + if end > len(rows) { + end = len(rows) + } + if err := ctx.Err(); err != nil { + return n, err + } + body := make([]byte, 0, (end-start)*128) + for _, r := range rows[start:end] { + line, err := rowLine(r) + if err != nil { + return n, err + } + body = append(body, line...) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, dest, bytes.NewReader(body)) + if err != nil { + return n, err + } + req.Header.Set("Content-Type", "application/x-ndjson") + if o.Install != "" { + req.Header.Set("X-Codeaf-Install", o.Install) + } + resp, err := client.Do(req) + if err != nil { + return n, err + } + ok := resp.StatusCode >= 200 && resp.StatusCode < 300 + io.CopyN(io.Discard, resp.Body, 1<<16) + resp.Body.Close() + if !ok { + return n, fmt.Errorf("outbox: %s replied %s", dest, resp.Status) + } + o.retire(sentMarks(rowNonces(rows[start:end]))) + n += end - start + } + return n, nil +} + +// sendFile appends the rows to the file at path, creating it with mode 0600 +// and its parent directory with mode 0700 if either is missing. +func (o *Outbox) sendFile(ctx context.Context, path string, rows []Row) (int, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + if err := os.MkdirAll(filepath.Dir(path), dirMode); err != nil { + return 0, err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, fileMode) + if err != nil { + return 0, err + } + defer f.Close() + body := make([]byte, 0, len(rows)*128) + for _, r := range rows { + line, err := rowLine(r) + if err != nil { + return 0, err + } + body = append(body, line...) + } + if _, err := f.Write(body); err != nil { + return 0, err + } + o.retire(sentMarks(rowNonces(rows))) + return len(rows), nil +} + +// compact checks that payload is valid JSON and answers it compacted, so a +// row is always one line. +func compact(payload []byte) ([]byte, error) { + if len(payload) == 0 { + return nil, errors.New("outbox: payload is not valid JSON") + } + var buf bytes.Buffer + if err := json.Compact(&buf, payload); err != nil { + return nil, fmt.Errorf("outbox: payload is not valid JSON: %w", err) + } + return buf.Bytes(), nil +} + +// rowLine answers the JSON object for one row as it is stored in the file, +// newline included. HTML escaping is off so a payload keeps the bytes compaction +// gave it, and every later read of the same row answers the same bytes. +func rowLine(row Row) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(row); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func rowNonces(rows []Row) []string { + nonces := make([]string, len(rows)) + for i, r := range rows { + nonces[i] = r.Nonce + } + return nonces +} + +func sentMarks(nonces []string) []mark { + return nonceMarks(nonces, true) +} + +func droppedMarks(nonces []string) []mark { + return nonceMarks(nonces, false) +} + +func nonceMarks(nonces []string, sent bool) []mark { + marks := make([]mark, len(nonces)) + for i, n := range nonces { + if sent { + marks[i] = mark{Sent: n} + } else { + marks[i] = mark{Dropped: n} + } + } + return marks +} diff --git a/internal/pool/outbox/outbox_test.go b/internal/pool/outbox/outbox_test.go new file mode 100644 index 0000000000..36b81e524f --- /dev/null +++ b/internal/pool/outbox/outbox_test.go @@ -0,0 +1,1131 @@ +package outbox + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// fixedClock answers a clock standing on the named UTC day. +func fixedClock(day string) func() time.Time { + t, err := time.Parse("2006-01-02", day) + if err != nil { + panic(err) + } + return func() time.Time { return t } +} + +// stepRand answers 16 fresh bytes per read, the first byte carrying step and +// the rest a counter, so every nonce a test appends is its own. +type stepRand struct { + step byte + n uint64 +} + +func (r *stepRand) Read(p []byte) (int, error) { + if len(p) < 16 { + return 0, io.ErrShortBuffer + } + p[0] = r.step + binary.LittleEndian.PutUint64(p[1:9], r.n) + copy(p[9:16], "outbox+") + r.n++ + return 16, nil +} + +// shortRand answers fewer bytes than a nonce needs. +type shortRand struct{ n int } + +func (r *shortRand) Read(p []byte) (int, error) { + if r.n == 0 { + return 0, io.EOF + } + n := len(p) + if n > r.n { + n = r.n + } + for i := 0; i < n; i++ { + p[i] = 0xaa + } + r.n -= n + return n, nil +} + +type errReader struct{} + +func (errReader) Read([]byte) (int, error) { return 0, errors.New("no bytes today") } + +func openOutbox(t *testing.T) (*Outbox, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "outbox.jsonl") + o, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { o.Close() }) + return o, path +} + +func appendRow(t *testing.T, o *Outbox, payload string) { + t.Helper() + if err := o.Append(json.RawMessage(payload)); err != nil { + t.Fatalf("Append(%s): %v", payload, err) + } +} + +func readAll(t *testing.T, path string) []byte { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return b +} + +func fileLines(t *testing.T, path string) []string { + t.Helper() + text := strings.TrimSuffix(string(readAll(t, path)), "\n") + if text == "" { + return nil + } + return strings.Split(text, "\n") +} + +func countMarks(t *testing.T, path, kind string) int { + t.Helper() + return strings.Count(string(readAll(t, path)), `{"`+kind+`":"`) +} + +func pendingNonces(t *testing.T, o *Outbox) []string { + t.Helper() + rows := o.Pending() + nonces := make([]string, len(rows)) + for i, r := range rows { + nonces[i] = r.Nonce + } + return nonces +} + +func mustEqual(t *testing.T, what string, got, want []byte) { + t.Helper() + if !bytes.Equal(got, want) { + t.Fatalf("%s: got %q, want %q", what, got, want) + } +} + +// postRecord answers every POST with 200 and records the body and the +// content-type under its own lock. A seconds field sleeps first; a status +// list answers its entries in request order before falling back to 200. +type postRecord struct { + mu sync.Mutex + bodies []string + types []string + headers []*http.Header + seconds int + status []int +} + +func (p *postRecord) handle(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + p.mu.Lock() + p.bodies = append(p.bodies, string(b)) + p.types = append(p.types, r.Header.Get("Content-Type")) + h := r.Header + p.headers = append(p.headers, &h) + // The two knobs are read under the same lock the bodies are written + // under: the server answers on its own goroutine, and a handler reading a + // field the test writes is a race whether or not the requests overlap. + wait := time.Duration(p.seconds) * time.Millisecond + status := 200 + if len(p.status) > 0 { + status = p.status[0] + p.status = p.status[1:] + } + p.mu.Unlock() + if wait > 0 { + time.Sleep(wait) + } + w.WriteHeader(status) +} + +func TestOpenRefusesAnEmptyPath(t *testing.T) { + if _, err := Open(""); err == nil { + t.Fatal(`Open("") answered no error`) + } +} + +func TestAFreshRunLeavesMode0600OnTheFileAnd0700OnTheDirectory(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing", "nested", "outbox.jsonl") + if _, err := Open(path); err != nil { + t.Fatalf("Open: %v", err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatalf("stat file: %v", err) + } + if fi.Mode().Perm() != 0600 { + t.Fatalf("file mode is %v, want 0600", fi.Mode().Perm()) + } + di, err := os.Stat(filepath.Dir(path)) + if err != nil { + t.Fatalf("stat dir: %v", err) + } + if di.Mode().Perm() != 0700 { + t.Fatalf("directory mode is %v, want 0700", di.Mode().Perm()) + } +} + +func TestOpenKeepsExistingRowsAndSkipsLinesThatAreNotRows(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.jsonl") + nonce := func(n int) string { return fmt.Sprintf("%032x", n) } + row1 := `{"schema":1,"day":"2026-09-16","nonce":"` + nonce(1) + `","payload":{"k":1}}` + row2 := `{"schema":1,"day":"2026-09-16","nonce":"` + nonce(2) + `","payload":{"k":2}}` + row3 := `{"schema":1,"day":"2026-09-17","nonce":"` + nonce(3) + `","payload":{"k":3}}` + row4 := `{"schema":1,"day":"2026-09-17","nonce":"` + nonce(4) + `","payload":{"k":4}}` + file := strings.Join([]string{ + row1, + "not json at all", + "null", + "[]", + `{"x":1}`, + row2, + `{"sent":"` + nonce(1) + `"}`, + `{"dropped":"` + nonce(3) + `"}`, + row3, // a marker may sit before the row it retires + row4, // unterminated last line + }, "\n") + if err := os.WriteFile(path, []byte(file), 0600); err != nil { + t.Fatal(err) + } + o, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer o.Close() + want := []string{nonce(2), nonce(4)} + got := pendingNonces(t, o) + if len(got) != len(want) { + t.Fatalf("Pending answered %d rows, want %d", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("Pending[%d] nonce is %s, want %s", i, got[i], want[i]) + } + } +} + +func TestReopenKeepsPendingRowsInTheOrderTheyWereAppended(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.jsonl") + o, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + o.Rand = &stepRand{step: 1} + appendRow(t, o, `{"k":1}`) + appendRow(t, o, `{"k":2}`) + if err := o.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + o2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer o2.Close() + got := pendingNonces(t, o2) + if len(got) != 2 || got[0] == got[1] { + t.Fatalf("reopen answered %v, want the two rows in order", got) + } + first := got[0] + appendRow(t, o2, `{"k":3}`) + got = pendingNonces(t, o2) + if len(got) != 3 || got[0] != first { + t.Fatalf("the third append disturbed the order: %v", got) + } +} + +func TestAppendWritesExactlyTheExpectedLine(t *testing.T) { + o, path := openOutbox(t) + o.Now = fixedClock("2026-09-17") + o.Rand = bytes.NewReader([]byte{0x0f, 0x1e, 0x2d, 0x3c, 0x4b, 0x5a, 0x69, 0x78, 0x87, 0x96, 0xa5, 0xb4, 0xc3, 0xd2, 0xe1, 0xf0}) + appendRow(t, o, `{"k":1}`) + want := `{"schema":1,"day":"2026-09-17","nonce":"0f1e2d3c4b5a69788796a5b4c3d2e1f0","payload":{"k":1}}` + "\n" + mustEqual(t, "file", readAll(t, path), []byte(want)) +} + +func TestAppendWritesTheUTCDayWhenTheClockSitsInAnotherZone(t *testing.T) { + o, path := openOutbox(t) + zone := time.FixedZone("test+0530", 5*3600+1800) + local := time.Date(2026, 9, 18, 2, 0, 0, 0, zone) // UTC: 2026-09-17 20:30 + o.Now = func() time.Time { return local } + o.Rand = &stepRand{step: 1} + appendRow(t, o, `{"k":1}`) + lines := fileLines(t, path) + if !strings.Contains(lines[0], `"day":"2026-09-17"`) { + t.Fatalf("row day is not the UTC day: %s", lines[0]) + } +} + +func TestAppendStoresThePayloadCompactedOnOneLine(t *testing.T) { + o, path := openOutbox(t) + o.Rand = &stepRand{step: 2} + appendRow(t, o, "{\n \"k\": 1,\n \"j\": [ 1 , 2 ]\n}") + lines := fileLines(t, path) + if len(lines) != 1 { + t.Fatalf("file holds %d lines, want 1", len(lines)) + } + if !strings.Contains(lines[0], `"payload":{"k":1,"j":[1,2]}`) { + t.Fatalf("payload was not stored compacted: %s", lines[0]) + } + rows := o.Pending() + if len(rows) != 1 || string(rows[0].Payload) != `{"k":1,"j":[1,2]}` { + t.Fatalf("Pending answered payload %v", rows) + } +} + +func TestAppendsBelowTheCapWriteOneLineEach(t *testing.T) { + o, path := openOutbox(t) + o.Rand = &stepRand{step: 3} + for i := 0; i < 10; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + } + if lines := fileLines(t, path); len(lines) != 10 { + t.Fatalf("file holds %d lines, want 10 and no marker lines", len(lines)) + } +} + +func TestAppendRefusesPayloadsItCannotStoreAndWritesNothing(t *testing.T) { + cases := []struct { + name string + payload string + }{ + {"empty", ``}, + {"open brace", `{`}, + {"bare word", `truthy`}, + {"two values", `{"a":1} {"b":2}`}, + {"truncated array", `[1,2`}, + {"newline inside the value", "[\"a\n\"]"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + o, path := openOutbox(t) + o.Rand = &stepRand{step: 4} + appendRow(t, o, `{"k":1}`) + before := readAll(t, path) + err := o.Append(json.RawMessage(c.payload)) + if err == nil { + t.Fatalf("Append accepted %q", c.payload) + } + mustEqual(t, "file", readAll(t, path), before) + if got := len(o.Pending()); got != 1 { + t.Fatalf("Pending answers %d rows after a refused append, want 1", got) + } + }) + } +} + +func TestAppendAppliesThePayloadCapOnceCompacted(t *testing.T) { + o, path := openOutbox(t) + o.Rand = &stepRand{step: 5} + // {"k":"aaa…"} compacts to a payload of len 8 + n. + filler := strings.Repeat("a", 65528) + if err := o.Append(json.RawMessage(`{"k":"` + filler + `"}`)); err != nil { + t.Fatalf("a payload of exactly 64 KiB once compacted was refused: %v", err) + } + if err := o.Append(json.RawMessage(`{"k":"` + filler + `a"}`)); err == nil { + t.Fatal("a payload one byte over 64 KiB once compacted was kept") + } + if lines := fileLines(t, path); len(lines) != 1 { + t.Fatalf("a refused append left %d lines, want the file untouched at 1", len(lines)) + } +} + +func TestAppendRefusesWhenTheNonceCannotBeRead(t *testing.T) { + cases := []struct { + name string + rand io.Reader + }{ + {"short read", &shortRand{n: 15}}, + {"failing reader", errReader{}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + o, path := openOutbox(t) + appendRow(t, o, `{"k":1}`) // crypto/rand answers this one + before := readAll(t, path) + o.Rand = c.rand + if err := o.Append(json.RawMessage(`{"k":2}`)); err == nil { + t.Fatal("Append answered no error for an unreadable nonce") + } + mustEqual(t, "file", readAll(t, path), before) + }) + } +} + +func TestAppendSurvivesManyGoroutinesWithoutLosingOrMixingRows(t *testing.T) { + o, path := openOutbox(t) + const goroutines, per = 32, 25 + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < per; i++ { + if err := o.Append(json.RawMessage(fmt.Sprintf(`{"g":%d,"i":%d}`, g, i))); err != nil { + t.Errorf("Append: %v", err) + return + } + } + }(g) + } + wg.Wait() + lines := fileLines(t, path) + if len(lines) != goroutines*per { + t.Fatalf("file holds %d lines, want %d", len(lines), goroutines*per) + } + seen := make(map[string]bool, len(lines)) + for _, line := range lines { + var r Row + if err := json.Unmarshal([]byte(line), &r); err != nil || r.Nonce == "" { + t.Fatalf("line does not parse on its own: %q (%v)", line, err) + } + seen[r.Nonce] = true + } + if len(seen) != len(lines) { + t.Fatalf("file holds %d distinct nonces across %d rows", len(seen), len(lines)) + } + if got := len(o.Pending()); got != len(lines) { + t.Fatalf("Pending answers %d rows, want %d", got, len(lines)) + } +} + +func TestPendingAnswersNothingForAFileItCannotRead(t *testing.T) { + path := filepath.Join(t.TempDir(), "outbox.jsonl") + o, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer o.Close() + o.Rand = &stepRand{step: 6} + appendRow(t, o, `{"k":1}`) + if got := len(o.Pending()); got != 1 { + t.Fatalf("Pending answers %d rows while the file is readable, want 1", got) + } + // A file that is gone, and one that turned into a directory, both read as + // no rows, and neither fails. + if err := os.Remove(path); err != nil { + t.Fatalf("remove: %v", err) + } + if got := len(o.Pending()); got != 0 { + t.Fatalf("Pending answered %d rows for a missing file, want 0", got) + } + if err := os.Mkdir(path, 0700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if got := len(o.Pending()); got != 0 { + t.Fatalf("Pending answered %d rows for an unreadable file, want 0", got) + } +} + +func TestTheCapDropsPendingRowsFromTheOldEnd(t *testing.T) { + o, path := openOutbox(t) + o.MaxPending = 3 + o.Rand = &stepRand{step: 7} + nonces := make([]string, 0, 5) + for i := 0; i < 5; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + rows := o.Pending() + nonces = append(nonces, rows[len(rows)-1].Nonce) + } + got := pendingNonces(t, o) + if len(got) != 3 || got[0] != nonces[2] || got[2] != nonces[4] { + t.Fatalf("Pending answers the wrong end: got %v, want %v..%v", got, nonces[2], nonces[4]) + } + if n := countMarks(t, path, "dropped"); n != 2 { + t.Fatalf("file holds %d dropped markers, want 2", n) + } + // The dropped rows stay gone across a reopen. + if err := o.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + o2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer o2.Close() + after := pendingNonces(t, o2) + if len(after) != 3 || after[0] != got[0] { + t.Fatalf("a reopen resurrected dropped rows: %v", after) + } +} + +func TestTheCapDefaultsTo5000AndAppliesToNegativeValues(t *testing.T) { + if DefaultMaxPending != 5000 { + t.Fatalf("DefaultMaxPending is %d, want 5000", DefaultMaxPending) + } + for _, max := range []int{0, -1} { + t.Run(fmt.Sprintf("MaxPending=%d", max), func(t *testing.T) { + o, path := openOutbox(t) + o.Rand = &stepRand{step: 7} + o.MaxPending = max + for i := 0; i < DefaultMaxPending+1; i++ { + if err := o.Append(json.RawMessage(fmt.Sprintf(`{"k":%d}`, i))); err != nil { + t.Fatalf("Append: %v", err) + } + } + if got := len(o.Pending()); got != DefaultMaxPending { + t.Fatalf("Pending answers %d rows, want %d", got, DefaultMaxPending) + } + if n := countMarks(t, path, "dropped"); n != 1 { + t.Fatalf("file holds %d dropped markers, want 1", n) + } + got := pendingNonces(t, o) + if got[0] == got[len(got)-1] { + t.Fatal("the cap did not drop from the old end") + } + }) + } +} + +func TestTheRowJustAppendedSurvivesACapOfOne(t *testing.T) { + o, _ := openOutbox(t) + o.MaxPending = 1 + o.Rand = &stepRand{step: 7} + appendRow(t, o, `{"k":1}`) + appendRow(t, o, `{"k":2}`) + got := o.Pending() + if len(got) != 1 || string(got[0].Payload) != `{"k":2}` { + t.Fatalf("Pending answers %v, want just the last row", got) + } +} + +func TestSendWithAnEmptyDestIsANoop(t *testing.T) { + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 8} + appendRow(t, o, `{"k":1}`) + appendRow(t, o, `{"k":2}`) + n, err := o.Send(context.Background(), "") + if n != 0 || err != nil { + t.Fatalf(`Send("") answered (%d, %v), want (0, nil)`, n, err) + } + if got := len(o.Pending()); got != 2 { + t.Fatalf("Pending answers %d rows after an empty dest, want 2", got) + } +} + +func TestSendWithNothingPendingMakesNoRequest(t *testing.T) { + rec := &postRecord{} + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, _ := openOutbox(t) + n, err := o.Send(context.Background(), srv.URL) + if n != 0 || err != nil { + t.Fatalf("Send answered (%d, %v), want (0, nil)", n, err) + } + if len(rec.bodies) != 0 { + t.Fatalf("a Send with nothing pending made %d requests", len(rec.bodies)) + } +} + +func TestSendPostsBatchesOfAtMost200Rows(t *testing.T) { + rec := &postRecord{} + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, path := openOutbox(t) + o.Rand = &stepRand{step: 9} + for i := 0; i < 450; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + } + rows := fileLines(t, path)[:450] // the rows, before any marker lines + n, err := o.Send(context.Background(), srv.URL) + if n != 450 || err != nil { + t.Fatalf("Send answered (%d, %v), want (450, nil)", n, err) + } + if len(rec.bodies) != 3 { + t.Fatalf("Send made %d posts, want 3", len(rec.bodies)) + } + wantLines := []int{200, 200, 50} + for i, want := range wantLines { + body := rec.bodies[i] + if got := strings.Count(body, "\n"); got != want { + t.Fatalf("post %d carries %d lines, want %d", i, got, want) + } + if !strings.HasSuffix(body, "\n") { + t.Fatalf("post %d does not end in a newline", i) + } + if rec.types[i] != "application/x-ndjson" { + t.Fatalf("post %d content-type is %q", i, rec.types[i]) + } + lo := i * 200 + hi := min(lo+200, 450) + mustEqual(t, fmt.Sprintf("post %d body", i), []byte(body), []byte(strings.Join(rows[lo:hi], "\n")+"\n")) + } + if got := o.Pending(); len(got) != 0 { + t.Fatalf("Pending answers %d rows after a full send, want 0", len(got)) + } + if err := o.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + o2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer o2.Close() + if got := pendingNonces(t, o2); len(got) != 0 { + t.Fatalf("the sent marks did not survive a reopen: %d rows pending", len(got)) + } +} + +func TestSendReachesAnHTTPSDestination(t *testing.T) { + rec := &postRecord{} + srv := httptest.NewTLSServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, _ := openOutbox(t) + o.Client = srv.Client() + o.Rand = &stepRand{step: 10} + appendRow(t, o, `{"k":1}`) + dest := strings.Replace(srv.URL, "http://", "https://", 1) + n, err := o.Send(context.Background(), dest) + if n != 1 || err != nil { + t.Fatalf("Send(%s) answered (%d, %v), want (1, nil)", dest, n, err) + } + if len(rec.bodies) != 1 { + t.Fatal("the https post never arrived") + } +} + +func TestSendKeepsUnarrivedRowsAndNeverSendsAnArrivedRowTwice(t *testing.T) { + seen := make(map[string]int) + var mu sync.Mutex + note := func(body string) { + mu.Lock() + defer mu.Unlock() + for _, line := range strings.Split(strings.TrimSuffix(body, "\n"), "\n") { + var r Row + if json.Unmarshal([]byte(line), &r) == nil && r.Nonce != "" { + seen[r.Nonce]++ + } + } + } + fail := &postRecord{status: []int{200, 500}} + srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + note(string(b)) + fail.handle(w, r) + })) + defer srv1.Close() + o, path := openOutbox(t) + o.Rand = &stepRand{step: 11} + for i := 0; i < 250; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + } + n, err := o.Send(context.Background(), srv1.URL) + if n != 200 || err == nil { + t.Fatalf("Send answered (%d, %v), want (200, an error)", n, err) + } + if cn := countMarks(t, path, "sent"); cn != 200 { + t.Fatalf("file holds %d sent markers after the first batch arrived, want 200", cn) + } + remaining := pendingNonces(t, o) + if len(remaining) != 50 { + t.Fatalf("%d rows stayed pending, want 50", len(remaining)) + } + ok := &postRecord{} + srv2 := httptest.NewServer(http.HandlerFunc(ok.handle)) + defer srv2.Close() + n, err = o.Send(context.Background(), srv2.URL) + if n != 50 || err != nil { + t.Fatalf("the resume answered (%d, %v), want (50, nil)", n, err) + } + for _, body := range ok.bodies { + note(body) + } + mu.Lock() + defer mu.Unlock() + if len(seen) != 250 { + t.Fatalf("%d distinct rows were sent, want 250", len(seen)) + } + for nonce, count := range seen { + // The 200 rows whose first batch answered 2xx were transmitted once + // and never again; the 50 rows the 500 refused were transmitted + // twice — once refused, once accepted — and never a third time. + want := 1 + for _, n2 := range remaining { + if n2 == nonce { + want = 2 + break + } + } + if count != want { + t.Fatalf("nonce %s… was sent %d times, want %d", nonce[:8], count, want) + } + } + if len(remaining) != 50 { + t.Fatal("the pending list was consumed by the check") + } +} + +func TestSendSaysWhatFailedAndLeavesTheRowsPending(t *testing.T) { + rec := &postRecord{status: []int{404}} + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 12} + appendRow(t, o, `{"k":1}`) + n, err := o.Send(context.Background(), srv.URL) + if n != 0 || err == nil { + t.Fatalf("Send answered (%d, %v), want (0, an error)", n, err) + } + if !strings.Contains(err.Error(), "404") { + t.Fatalf("the error does not say what failed: %v", err) + } + if got := len(o.Pending()); got != 1 { + t.Fatalf("%d rows stayed pending after the refusal, want 1", got) + } +} + +func TestSendAnswersARefusedConnection(t *testing.T) { + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 12} + appendRow(t, o, `{"k":1}`) + n, err := o.Send(context.Background(), "http://127.0.0.1:1/") + if n != 0 || err == nil { + t.Fatalf("Send answered (%d, %v), want (0, an error)", n, err) + } + if got := len(o.Pending()); got != 1 { + t.Fatalf("%d rows stayed pending, want 1", got) + } +} + +func TestSendLivesInsideTheBudget(t *testing.T) { + rec := &postRecord{seconds: 900} + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 13} + o.Budget = 150 * time.Millisecond + for i := 0; i < 5; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + } + n, err := o.Send(context.Background(), srv.URL) + if n != 0 || err == nil { + t.Fatalf("Send answered (%d, %v), want (0, an error)", n, err) + } + if got := len(o.Pending()); got != 5 { + t.Fatalf("%d rows stayed pending, want 5", got) + } +} + +func TestSendLivesInsideTheContextWhenItEndsFirst(t *testing.T) { + rec := &postRecord{seconds: 900} + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 14} + o.Budget = time.Minute + o.Rand = &stepRand{step: 14} + appendRow(t, o, `{"k":1}`) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + if _, err := o.Send(ctx, srv.URL); err == nil { + t.Fatal("Send outlived its context") + } + if got := len(o.Pending()); got != 1 { + t.Fatalf("%d rows stayed pending, want 1", got) + } +} + +func TestSendWithACanceledContextMakesNoRequest(t *testing.T) { + rec := &postRecord{} + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 15} + appendRow(t, o, `{"k":1}`) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + n, err := o.Send(ctx, srv.URL) + if n != 0 || err == nil { + t.Fatalf("Send answered (%d, %v), want (0, an error)", n, err) + } + if len(rec.bodies) != 0 { + t.Fatal("a canceled context still made a request") + } +} + +func TestSendAppendsTheRowsToFileDestination(t *testing.T) { + dest := filepath.Join(t.TempDir(), "missing", "sub", "out.ndjson") + o, path := openOutbox(t) + o.Rand = &stepRand{step: 16} + for i := 0; i < 3; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + } + rows := fileLines(t, path) + n, err := o.Send(context.Background(), dest) + if n != 3 || err != nil { + t.Fatalf("Send answered (%d, %v), want (3, nil)", n, err) + } + sent := fileLines(t, dest) + if len(sent) != 3 { + t.Fatalf("destination holds %d lines, want 3", len(sent)) + } + for i := range sent { + if sent[i] != rows[i] { + t.Fatalf("destination line %d is %q, want the row %q", i, sent[i], rows[i]) + } + } + fi, err := os.Stat(dest) + if err != nil { + t.Fatalf("stat dest: %v", err) + } + if fi.Mode().Perm() != 0600 { + t.Fatalf("destination mode is %v, want 0600", fi.Mode().Perm()) + } + di, err := os.Stat(filepath.Dir(dest)) + if err != nil { + t.Fatalf("stat dest dir: %v", err) + } + if di.Mode().Perm() != 0700 { + t.Fatalf("destination directory mode is %v, want 0700", di.Mode().Perm()) + } + if got := len(o.Pending()); got != 0 { + t.Fatalf("%d rows stayed pending after a file send, want 0", got) + } + if err := o.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + o2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer o2.Close() + if got := pendingNonces(t, o2); len(got) != 0 { + t.Fatalf("a reopen still answers %d rows, want none", len(got)) + } +} + +func TestSendAcceptsTheFileScheme(t *testing.T) { + dest := filepath.Join(t.TempDir(), "out.ndjson") + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 17} + appendRow(t, o, `{"k":1}`) + n, err := o.Send(context.Background(), "file://"+dest) + if n != 1 || err != nil { + t.Fatalf("Send answered (%d, %v), want (1, nil)", n, err) + } + if lines := fileLines(t, dest); len(lines) != 1 { + t.Fatalf("destination holds %d lines, want 1", len(lines)) + } +} + +func TestSendAppendsToFileAcrossSends(t *testing.T) { + dest := filepath.Join(t.TempDir(), "out.ndjson") + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 17} + appendRow(t, o, `{"k":1}`) + appendRow(t, o, `{"k":2}`) + if n, err := o.Send(context.Background(), dest); n != 2 || err != nil { + t.Fatalf("first send answered (%d, %v)", n, err) + } + appendRow(t, o, `{"k":3}`) + if n, err := o.Send(context.Background(), dest); n != 1 || err != nil { + t.Fatalf("second send answered (%d, %v)", n, err) + } + if lines := fileLines(t, dest); len(lines) != 3 || !strings.Contains(lines[2], `"k":3`) { + t.Fatalf("destination holds %v", lines) + } +} + +func TestKeepLeavesDeclinedRowsUnsentAndGone(t *testing.T) { + rec := &postRecord{} + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, path := openOutbox(t) + o.Rand = &stepRand{step: 18} + asked := 0 + o.Keep = func(payload json.RawMessage) bool { + asked++ + s := string(payload) + return !strings.Contains(s, `"k":2`) && !strings.Contains(s, `"k":4`) + } + for i := 1; i <= 4; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + } + n, err := o.Send(context.Background(), srv.URL) + if n != 2 || err != nil { + t.Fatalf("Send answered (%d, %v), want (2, nil)", n, err) + } + if asked != 4 { + t.Fatalf("Keep was asked %d times, want 4", asked) + } + body := strings.Join(rec.bodies, "") + if strings.Contains(body, `"k":2`) || strings.Contains(body, `"k":4`) { + t.Fatal("a declined payload reached the destination") + } + if got := len(o.Pending()); got != 0 { + t.Fatalf("%d rows stayed pending, want 0 — a declined row leaves the outbox", got) + } + if err := o.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + o2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer o2.Close() + if got := pendingNonces(t, o2); len(got) != 0 { + t.Fatalf("a declined row came back after a reopen: %d rows", len(got)) + } +} + +func TestKeepDecliningEverythingSendsNothing(t *testing.T) { + rec := &postRecord{} + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 19} + o.Keep = func(json.RawMessage) bool { return false } + appendRow(t, o, `{"k":1}`) + n, err := o.Send(context.Background(), srv.URL) + if n != 0 || err != nil { + t.Fatalf("Send answered (%d, %v), want (0, nil)", n, err) + } + if len(rec.bodies) != 0 { + t.Fatal("a Send that declines everything still made a request") + } + if got := len(o.Pending()); got != 0 { + t.Fatalf("%d rows stayed pending, want 0", got) + } +} + +func TestRowsAppendedAfterASendArePendingLikeAnyOther(t *testing.T) { + dest := filepath.Join(t.TempDir(), "out.ndjson") + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 19} + appendRow(t, o, `{"k":1}`) + if n, err := o.Send(context.Background(), dest); n != 1 || err != nil { + t.Fatalf("first send answered (%d, %v)", n, err) + } + appendRow(t, o, `{"k":2}`) + got := o.Pending() + if len(got) != 1 || string(got[0].Payload) != `{"k":2}` { + t.Fatalf("Pending answers %v, want the fresh row", got) + } +} + +func TestSendRefusesNonsenseDestinationsWithoutPanicking(t *testing.T) { + cases := []string{ + "http://", + "https://", + "http://[::1", + "file://", + "\x00", + "file://\x00", + } + for _, dest := range cases { + t.Run(fmt.Sprintf("%q", dest), func(t *testing.T) { + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 20} + appendRow(t, o, `{"k":1}`) + n, err := o.Send(context.Background(), dest) + if err == nil || n != 0 { + t.Fatalf("Send(%q) answered (%d, %v), want (0, an error)", dest, n, err) + } + if got := len(o.Pending()); got != 1 { + t.Fatalf("%d rows stayed pending after the refusal, want 1", got) + } + }) + } +} + +func TestCloseIsIdempotentAndAClosedOutboxRefusesWork(t *testing.T) { + dest := filepath.Join(t.TempDir(), "out.ndjson") + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 21} + appendRow(t, o, `{"k":1}`) + appendRow(t, o, `{"k":2}`) + if err := o.Close(); err != nil { + t.Fatalf("first Close: %v", err) + } + if err := o.Close(); err != nil { + t.Fatalf("second Close: %v", err) + } + if err := o.Append(json.RawMessage(`{"k":3}`)); err == nil { + t.Fatal("a closed outbox accepted an append") + } + if n, err := o.Send(context.Background(), dest); err == nil || n != 0 { + t.Fatalf("a closed outbox sent (%d, %v)", n, err) + } + if got := len(o.Pending()); got != 2 { + t.Fatalf("Pending answers %d rows after Close, want 2", got) + } +} + +func TestFieldsSetAfterOpenAreUsed(t *testing.T) { + o, path := openOutbox(t) + o.Now = fixedClock("2026-09-17") + o.Rand = &stepRand{step: 21} + o.MaxPending = 2 + for i := 0; i < 4; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + } + got := o.Pending() + if len(got) != 2 { + t.Fatalf("the cap set after Open was ignored: %d rows pending", len(got)) + } + lines := fileLines(t, path) + if !strings.Contains(lines[0], `"day":"2026-09-17"`) { + t.Fatalf("the clock set after Open was ignored: %s", lines[0]) + } + if !strings.Contains(lines[0], `"nonce":"15`) { // stepRand answers the step as its first byte + t.Fatalf("the nonce reader set after Open was ignored: %s", lines[0]) + } +} + +func TestAppendWithTheDefaultClockAndRand(t *testing.T) { + o, _ := openOutbox(t) + before := time.Now().UTC().Format("2006-01-02") + for i := 0; i < 20; i++ { + appendRow(t, o, fmt.Sprintf(`{"k":%d}`, i)) + } + after := time.Now().UTC().Format("2006-01-02") + for i, r := range o.Pending() { + if len(r.Nonce) != 32 { + t.Fatalf("row %d nonce is %d characters, want 32", i, len(r.Nonce)) + } + if _, err := hex.DecodeString(r.Nonce); err != nil { + t.Fatalf("row %d nonce is not lowercase hex: %v", i, err) + } + if r.Nonce != strings.ToLower(r.Nonce) { + t.Fatalf("row %d nonce is not lowercase", i) + } + if r.Day != before && r.Day != after { + t.Fatalf("row %d day is %s, want the UTC day (%s or %s)", i, r.Day, before, after) + } + if r.Schema != 1 { + t.Fatalf("row %d schema is %d, want 1", i, r.Schema) + } + } +} + +// A nonce is what the caller's own Rand made of sixteen bytes, so a source +// that answers with the same bytes twice writes the same nonce twice. Each +// marker still retires exactly one row. +func TestRowsSharingANonceAreRetiredOneMarkerAtATime(t *testing.T) { + o, _ := openOutbox(t) + o.Rand = bytes.NewReader(bytes.Repeat([]byte{3}, 4096)) + o.MaxPending = 3 + for i := 0; i < 5; i++ { + appendRow(t, o, fmt.Sprintf(`{"i":%d}`, i)) + } + pending := o.Pending() + if len(pending) != 3 { + t.Fatalf("pending = %d rows, want the three the cap leaves", len(pending)) + } + for i, r := range pending { + if want := fmt.Sprintf(`{"i":%d}`, i+2); string(r.Payload) != want { + t.Fatalf("pending[%d] = %s, want %s: the cap drops from the old end", i, r.Payload, want) + } + } +} + +// A partial send over a shared nonce leaves the rows it never reached, rather +// than retiring them with the batch that did arrive. +func TestAPartialSendOverASharedNonceKeepsTheRowsItDidNotReach(t *testing.T) { + post := &postRecord{} + srv := httptest.NewServer(http.HandlerFunc(post.handle)) + defer srv.Close() + + o, _ := openOutbox(t) + o.Rand = bytes.NewReader(bytes.Repeat([]byte{5}, 4096)) + o.Client = srv.Client() + for i := 0; i < 250; i++ { + appendRow(t, o, fmt.Sprintf(`{"i":%d}`, i)) + } + post.status = []int{200, 500} + + n, err := o.Send(context.Background(), srv.URL) + if err == nil { + t.Fatal("a refused second batch returned no error") + } + if n != 200 { + t.Fatalf("Send = %d, want the 200 rows the first batch carried", n) + } + if got := len(o.Pending()); got != 50 { + t.Fatalf("pending = %d, want the 50 rows the refused batch never delivered", got) + } +} + +// A marker that could not be written leaves its rows pending: a row gone from +// the pending set and present in the file is a row that comes back at the next +// Open having been reported delivered. +func TestASendWhoseMarkerCannotBeWrittenKeepsTheRowsPending(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "outbox.jsonl") + o, err := Open(path) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer o.Close() + appendRow(t, o, `{"i":0}`) + + sink := filepath.Join(dir, "sink.jsonl") + // The handle goes away between the delivery and the marker, which is what + // a Close racing an in-flight Send does. + o.Keep = func(json.RawMessage) bool { + o.Close() + return true + } + if _, err := o.Send(context.Background(), sink); err != nil { + t.Fatalf("Send: %v", err) + } + if got := len(o.Pending()); got != 1 { + t.Fatalf("pending = %d, want the row whose marker never reached the file", got) + } +} + +// A batch sent over http carries the outbox's install nonce as +// X-Codeaf-Install, one header per post; an outbox with none set sends no +// header, and a file destination is never asked for one. +func TestSendCarriesTheInstallNonceAsAHeader(t *testing.T) { + var rec postRecord + srv := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv.Close() + + o, _ := openOutbox(t) + o.Rand = &stepRand{step: 11} + o.Install = "0123456789abcdef0123456789abcdef" + appendRow(t, o, `{"k":1}`) + n, err := o.Send(context.Background(), srv.URL) + if n != 1 || err != nil { + t.Fatalf("Send answered (%d, %v), want (1, nil)", n, err) + } + if got := rec.headers[0].Get("X-Codeaf-Install"); got != o.Install { + t.Fatalf("the batch rode with X-Codeaf-Install %q, want the outbox's nonce %q", got, o.Install) + } + + rec = postRecord{} + srv2 := httptest.NewServer(http.HandlerFunc(rec.handle)) + defer srv2.Close() + o2, _ := openOutbox(t) + o2.Rand = &stepRand{step: 12} + appendRow(t, o2, `{"k":1}`) + if _, err := o2.Send(context.Background(), srv2.URL); err != nil { + t.Fatalf("Send: %v", err) + } + if got := rec.headers[0].Get("X-Codeaf-Install"); got != "" { + t.Fatalf("an outbox with no nonce sent the header anyway: %q", got) + } +} diff --git a/internal/pool/poolcfg/poolcfg.go b/internal/pool/poolcfg/poolcfg.go new file mode 100644 index 0000000000..4680f72c51 --- /dev/null +++ b/internal/pool/poolcfg/poolcfg.go @@ -0,0 +1,307 @@ +// Package poolcfg resolves how the measurement pool behaves from two stored +// settings — the mode word and the trusted key — and an environment handed in +// as a function. Nothing here touches the disk, the network, the clock or the +// process environment, so Resolve is a table of inputs and outputs and every +// case is reachable from a test. +package poolcfg + +import ( + "fmt" + "net/url" + "strings" + "time" +) + +// Addresses held when the environment offers nothing readable. The relay is +// the one base the resolver holds: the index and the submit addresses are +// read from beside it, and each keeps its own override. The mirror is a +// second index address — the relay's document copied to GitHub — held on its +// own, because it derives from nothing here. +const ( + DefaultRelayURL = "https://codeaf.agentfield.ai/pool" + DefaultIndexURL = DefaultRelayURL + indexPath + DefaultSubmitURL = DefaultRelayURL + submitPath + DefaultMirrorURL = "https://raw.githubusercontent.com/Agent-Field/CodeAF/model-pool/pool/index.json" +) + +// The two addresses a relay serves, under its base. +const ( + indexPath = "/index.json" + submitPath = "/v1/rows" +) + +// Mode says what the pool may do: send and read, only read, or neither. +type Mode int + +const ( + On Mode = iota + Read + Off +) + +// String spells the mode the way the setting and the environment spell it. +func (m Mode) String() string { + switch m { + case On: + return "on" + case Read: + return "read" + case Off: + return "off" + default: + return fmt.Sprintf("Mode(%d)", int(m)) + } +} + +// Sources records where each resolved value came from. Every field holds one +// of four words: "default", "setting", "env" or "ci". +type Sources struct { + Mode string + RelayURL string + IndexURL string + SubmitURL string + MirrorURL string + PublicKey string + TTL string +} + +// Config is the resolved pool configuration. Source names the origin of each +// field, and every field is resolved on its own: the mode does not change how +// the addresses or the TTL are read, and an unreadable URL does not disturb +// the TTL. +// +// The relay is the base the index and submit addresses derive from: each is +// RelayURL with its own path appended, unless its own environment name +// overrides it. The mirror is a second index address held on its own, since +// it derives from nothing here; an empty one turns the fallback off. An empty +// PublicKey is the ordinary answer — it means the key the binary carries. +type Config struct { + Mode Mode + RelayURL string + IndexURL string + SubmitURL string + MirrorURL string + PublicKey string + TTL time.Duration + Source Sources +} + +// The eight names Resolve reads, and no others. +const ( + envMode = "CODEAF_MODEL_POOL" + envCI = "CI" + envRelay = "CODEAF_MODEL_POOL_RELAY_URL" + envIndex = "CODEAF_MODEL_POOL_URL" + envSubmit = "CODEAF_MODEL_POOL_SUBMIT_URL" + envMirror = "CODEAF_MODEL_POOL_MIRROR_URL" + envTTL = "CODEAF_MODEL_POOL_TTL" + envKey = "CODEAF_MODEL_POOL_PUBLIC_KEY" +) + +// Where a resolved value came from. +const ( + srcDefault = "default" + srcSetting = "setting" + srcEnv = "env" + srcCI = "ci" +) + +// TTL bounds, and the value held when no readable TTL is set. +const ( + ttlFloor = time.Minute + ttlCeiling = 7 * 24 * time.Hour + defaultTTL = 24 * time.Hour +) + +// Resolve resolves the pool configuration from the stored mode word, the +// stored public key and the environment behind lookup, which answers with a +// value and whether the name was set at all — a name that is set and empty is +// not the same as a name nobody set. A nil lookup is an environment in which +// nothing is set. +// +// Every value is read with the space around it trimmed, and words are matched +// without regard to case. Resolve is pure: the same inputs give the same +// Config, it keeps nothing between calls, and many goroutines may call it at +// once. +func Resolve(setting, publicKey string, lookup func(name string) (value string, set bool)) Config { + // One injection for the whole environment: the eight names are read once, + // here, in this order, and each value is trimmed as it is read. + get := func(name string) (string, bool) { + if lookup == nil { + return "", false + } + value, set := lookup(name) + if !set { + return "", false + } + return strings.TrimSpace(value), true + } + modeWord, _ := get(envMode) + ciWord, _ := get(envCI) + relayWord, _ := get(envRelay) + indexWord, _ := get(envIndex) + submitWord, submitSet := get(envSubmit) + mirrorWord, mirrorSet := get(envMirror) + ttlWord, _ := get(envTTL) + keyWord, _ := get(envKey) + + // The mode is the first of these that answers, and that answer is its + // source: the environment, the stored setting, CI, then the default. A + // word neither the environment nor the setting knows is simply not an + // answer and the next line gets its turn. + mode, modeSrc := On, srcDefault + if m, ok := parseMode(modeWord); ok { + mode, modeSrc = m, srcEnv + } else if m, ok := parseMode(strings.TrimSpace(setting)); ok { + mode, modeSrc = m, srcSetting + } else if ciSaysYes(ciWord) { + mode, modeSrc = Read, srcCI + } + + // The relay is the base the other two addresses derive from, and its + // source is theirs: a relay read from the environment hands down both of + // its derived addresses, each of which an override of its own still wins. + // A base is held with no trailing slash, whatever the name carried. + relayURL, relaySrc := DefaultRelayURL, srcDefault + if u, ok := acceptURL(relayWord); ok { + relayURL, relaySrc = strings.TrimRight(u, "/"), srcEnv + } + + indexURL, indexSrc := relayURL+indexPath, relaySrc + if u, ok := acceptURL(indexWord); ok { + indexURL, indexSrc = u, srcEnv + } + + // An empty submit address means send nowhere, and that is an answer; a + // name nobody set, or one holding neither an address nor emptiness, is + // the address derived from the relay. + submitURL, submitSrc := relayURL+submitPath, relaySrc + if submitSet { + if submitWord == "" { + submitURL, submitSrc = "", srcEnv + } else if u, ok := acceptURL(submitWord); ok { + submitURL, submitSrc = u, srcEnv + } + } + + // The mirror derives from nothing: it is the address held, or the one the + // environment names, or — a value set and empty — nothing at all, which + // turns the fallback off. + mirrorURL, mirrorSrc := DefaultMirrorURL, srcDefault + if mirrorSet { + if mirrorWord == "" { + mirrorURL, mirrorSrc = "", srcEnv + } else if u, ok := acceptURL(mirrorWord); ok { + mirrorURL, mirrorSrc = u, srcEnv + } + } + + // A readable TTL is held between the floor and the ceiling; a name nobody + // set, or one no duration can be read from, is the default. + ttl, ttlSrc := defaultTTL, srcDefault + if d, err := time.ParseDuration(ttlWord); err == nil { + ttl, ttlSrc = clampTTL(d), srcEnv + } + + // The trusted key is the first of these that answers: the environment, + // then the stored row, then none. An empty key is the ordinary answer — + // it means the key the binary carries. + storedKey := strings.TrimSpace(publicKey) + publicKey, keySrc := "", srcDefault + if keyWord != "" { + publicKey, keySrc = keyWord, srcEnv + } else if storedKey != "" { + publicKey, keySrc = storedKey, srcSetting + } + + return Config{ + Mode: mode, + RelayURL: relayURL, + IndexURL: indexURL, + SubmitURL: submitURL, + MirrorURL: mirrorURL, + PublicKey: publicKey, + TTL: ttl, + Source: Sources{ + Mode: modeSrc, + RelayURL: relaySrc, + IndexURL: indexSrc, + SubmitURL: submitSrc, + MirrorURL: mirrorSrc, + PublicKey: keySrc, + TTL: ttlSrc, + }, + } +} + +// CanSend reports whether the pool may send measurements: only when the mode +// is on and there is a submit address to send to. +func (c Config) CanSend() bool { + return c.Mode == On && c.SubmitURL != "" +} + +// CanRead reports whether the pool may read measurements: whenever the mode +// is not off. +func (c Config) CanRead() bool { + return c.Mode != Off +} + +// parseMode reads one of the three mode words, without regard to case. +// Anything else is not an answer. +func parseMode(word string) (Mode, bool) { + switch strings.ToLower(word) { + case "on": + return On, true + case "read": + return Read, true + case "off": + return Off, true + } + return On, false +} + +// ciSaysYes reads the CI word: one of true, 1 or yes, without regard to case. +func ciSaysYes(word string) bool { + switch strings.ToLower(word) { + case "true", "1", "yes": + return true + } + return false +} + +// acceptURL reads one of the four shapes a pool address may take, and nothing +// else: an "https://" URL, an "http://" URL whose host is localhost or +// 127.0.0.1 with or without a port, a "file://" URL, or an absolute filesystem +// path, which is one beginning "/". Plain http anywhere else is not accepted. +func acceptURL(value string) (string, bool) { + if strings.HasPrefix(value, "/") { + return value, true + } + lower := strings.ToLower(value) + switch { + case strings.HasPrefix(lower, "https://"), strings.HasPrefix(lower, "file://"): + if _, err := url.Parse(value); err != nil { + return "", false + } + return value, true + case strings.HasPrefix(lower, "http://"): + u, err := url.Parse(value) + if err != nil { + return "", false + } + host := u.Hostname() + return value, strings.EqualFold(host, "localhost") || host == "127.0.0.1" + } + return "", false +} + +// clampTTL holds d between the TTL floor and ceiling. +func clampTTL(d time.Duration) time.Duration { + if d < ttlFloor { + return ttlFloor + } + if d > ttlCeiling { + return ttlCeiling + } + return d +} diff --git a/internal/pool/poolcfg/poolcfg_test.go b/internal/pool/poolcfg/poolcfg_test.go new file mode 100644 index 0000000000..87ffcc5eb6 --- /dev/null +++ b/internal/pool/poolcfg/poolcfg_test.go @@ -0,0 +1,539 @@ +package poolcfg + +import ( + "reflect" + "sort" + "sync" + "testing" + "time" +) + +// envOf builds a lookup over a map. A name present with an empty value counts +// as set, because a name that is set and empty is not the same as a name +// nobody set. +func envOf(pairs map[string]string) func(string) (string, bool) { + return func(name string) (string, bool) { + value, set := pairs[name] + return value, set + } +} + +func TestModeString(t *testing.T) { + cases := []struct { + mode Mode + want string + }{ + {On, "on"}, + {Read, "read"}, + {Off, "off"}, + } + for _, c := range cases { + if got := c.mode.String(); got != c.want { + t.Errorf("Mode(%d).String() = %q, want %q", int(c.mode), got, c.want) + } + } +} + +func TestResolveDefaults(t *testing.T) { + want := Config{ + Mode: On, + RelayURL: DefaultRelayURL, + IndexURL: DefaultIndexURL, + SubmitURL: DefaultSubmitURL, + MirrorURL: DefaultMirrorURL, + PublicKey: "", + TTL: 24 * time.Hour, + Source: Sources{ + Mode: "default", + RelayURL: "default", + IndexURL: "default", + SubmitURL: "default", + MirrorURL: "default", + PublicKey: "default", + TTL: "default", + }, + } + lookups := []struct { + name string + lookup func(string) (string, bool) + }{ + {"nil lookup", nil}, + {"nobody set", envOf(nil)}, + {"other names only", envOf(map[string]string{"OTHER": "on"})}, + } + for _, l := range lookups { + if got := Resolve("", "", l.lookup); got != want { + t.Errorf("%s: Resolve() = %+v, want %+v", l.name, got, want) + } + } +} + +func TestResolveMode(t *testing.T) { + cases := []struct { + name string + setting string + lookup func(string) (string, bool) + want Mode + wantSrc string + }{ + {"env on", "", envOf(map[string]string{"CODEAF_MODEL_POOL": "on"}), On, "env"}, + {"env read", "", envOf(map[string]string{"CODEAF_MODEL_POOL": "read"}), Read, "env"}, + {"env off", "", envOf(map[string]string{"CODEAF_MODEL_POOL": "off"}), Off, "env"}, + {"env padded and mixed case", "", envOf(map[string]string{"CODEAF_MODEL_POOL": "\t OFF "}), Off, "env"}, + {"env unknown word falls through", "read", envOf(map[string]string{"CODEAF_MODEL_POOL": "maybe"}), Read, "setting"}, + {"env set and empty falls through", "read", envOf(map[string]string{"CODEAF_MODEL_POOL": ""}), Read, "setting"}, + {"setting on", "on", envOf(nil), On, "setting"}, + {"setting read", "read", envOf(nil), Read, "setting"}, + {"setting padded and mixed case", " Off\t", envOf(nil), Off, "setting"}, + {"setting unknown word falls through", "sometimes", envOf(nil), On, "default"}, + {"setting blank falls through", " ", envOf(nil), On, "default"}, + {"env beats setting", "off", envOf(map[string]string{"CODEAF_MODEL_POOL": "on"}), On, "env"}, + } + for _, c := range cases { + got := Resolve(c.setting, "", c.lookup) + if got.Mode != c.want || got.Source.Mode != c.wantSrc { + t.Errorf("%s: Mode = %v from %q, want %v from %q", c.name, got.Mode, got.Source.Mode, c.want, c.wantSrc) + } + } +} + +func TestResolveModeFromCI(t *testing.T) { + cases := []struct { + name string + setting string + lookup func(string) (string, bool) + want Mode + wantSrc string + }{ + {"ci true", "", envOf(map[string]string{"CI": "true"}), Read, "ci"}, + {"ci one", "", envOf(map[string]string{"CI": "1"}), Read, "ci"}, + {"ci yes", "", envOf(map[string]string{"CI": "yes"}), Read, "ci"}, + {"ci padded and mixed case", "", envOf(map[string]string{"CI": " YES "}), Read, "ci"}, + {"ci false is not an answer", "", envOf(map[string]string{"CI": "false"}), On, "default"}, + {"ci zero is not an answer", "", envOf(map[string]string{"CI": "0"}), On, "default"}, + {"ci no is not an answer", "", envOf(map[string]string{"CI": "no"}), On, "default"}, + {"ci set and empty is not an answer", "", envOf(map[string]string{"CI": ""}), On, "default"}, + {"ci reads when the setting is unknown", "maybe", envOf(map[string]string{"CI": "true"}), Read, "ci"}, + {"ci reads when the setting is blank", " ", envOf(map[string]string{"CI": "1"}), Read, "ci"}, + {"setting beats ci", "off", envOf(map[string]string{"CI": "true"}), Off, "setting"}, + {"env on beats ci", "off", envOf(map[string]string{"CI": "true", "CODEAF_MODEL_POOL": "on"}), On, "env"}, + {"env off beats ci", "on", envOf(map[string]string{"CI": "true", "CODEAF_MODEL_POOL": "off"}), Off, "env"}, + {"env unknown and setting unknown leave ci", "maybe", envOf(map[string]string{"CI": "true", "CODEAF_MODEL_POOL": "maybe"}), Read, "ci"}, + } + for _, c := range cases { + got := Resolve(c.setting, "", c.lookup) + if got.Mode != c.want || got.Source.Mode != c.wantSrc { + t.Errorf("%s: Mode = %v from %q, want %v from %q", c.name, got.Mode, got.Source.Mode, c.want, c.wantSrc) + } + } +} + +func TestResolveIndexURL(t *testing.T) { + cases := []struct { + name string + value string + set bool + want string + wantSrc string + }{ + {"nobody set", "", false, DefaultIndexURL, "default"}, + {"https", "https://pool.example.com/index.json", true, "https://pool.example.com/index.json", "env"}, + {"https with port", "https://pool.example.com:8443/i", true, "https://pool.example.com:8443/i", "env"}, + {"https scheme in upper case", "HTTPS://pool.example.com", true, "HTTPS://pool.example.com", "env"}, + {"http localhost", "http://localhost", true, "http://localhost", "env"}, + {"http localhost with port", "http://localhost:8080", true, "http://localhost:8080", "env"}, + {"http localhost in upper case", "http://LOCALHOST", true, "http://LOCALHOST", "env"}, + {"http loopback", "http://127.0.0.1", true, "http://127.0.0.1", "env"}, + {"http loopback with port and path", "http://127.0.0.1:9000/pool.json", true, "http://127.0.0.1:9000/pool.json", "env"}, + {"file", "file:///var/tmp/pool.json", true, "file:///var/tmp/pool.json", "env"}, + {"absolute path", "/var/tmp/pool.json", true, "/var/tmp/pool.json", "env"}, + {"padded", " https://pool.example.com ", true, "https://pool.example.com", "env"}, + {"plain http elsewhere", "http://pool.example.com", true, DefaultIndexURL, "default"}, + {"plain http elsewhere with port", "http://pool.example.com:8080", true, DefaultIndexURL, "default"}, + {"http host that merely contains localhost", "http://localhost.example.com", true, DefaultIndexURL, "default"}, + {"http loopback that is not 127.0.0.1", "http://127.0.0.2", true, DefaultIndexURL, "default"}, + // The host is the host the request would reach, so a loopback name in + // the userinfo or as a label of somebody else's name is not one. These + // rows pin the read: a shape check written against the raw text rather + // than the parsed host would take all three. + {"http loopback name in the userinfo", "http://localhost@pool.example.com", true, DefaultIndexURL, "default"}, + {"http loopback name and port in the userinfo", "http://localhost:80@pool.example.com", true, DefaultIndexURL, "default"}, + {"http loopback address as a leading label", "http://127.0.0.1.pool.example.com", true, DefaultIndexURL, "default"}, + {"other scheme", "ftp://pool.example.com", true, DefaultIndexURL, "default"}, + {"scheme without the double slash", "https:pool.example.com", true, DefaultIndexURL, "default"}, + {"http without the double slash", "http:/localhost", true, DefaultIndexURL, "default"}, + {"bare host", "pool.example.com/index.json", true, DefaultIndexURL, "default"}, + {"relative path", "tmp/pool.json", true, DefaultIndexURL, "default"}, + {"dot relative path", "./pool.json", true, DefaultIndexURL, "default"}, + {"set and empty", "", true, DefaultIndexURL, "default"}, + } + for _, c := range cases { + lookup := envOf(nil) + if c.set { + lookup = envOf(map[string]string{"CODEAF_MODEL_POOL_URL": c.value}) + } + got := Resolve("", "", lookup) + if got.IndexURL != c.want || got.Source.IndexURL != c.wantSrc { + t.Errorf("%s: IndexURL = %q from %q, want %q from %q", c.name, got.IndexURL, got.Source.IndexURL, c.want, c.wantSrc) + } + } +} + +func TestResolveSubmitURL(t *testing.T) { + cases := []struct { + name string + value string + set bool + want string + wantSrc string + }{ + {"nobody set", "", false, DefaultSubmitURL, "default"}, + {"https", "https://submit.example.com", true, "https://submit.example.com", "env"}, + {"http loopback with port", "http://127.0.0.1:9000/submit", true, "http://127.0.0.1:9000/submit", "env"}, + {"file", "file:///tmp/submit.json", true, "file:///tmp/submit.json", "env"}, + {"absolute path", "/tmp/submit.json", true, "/tmp/submit.json", "env"}, + {"empty sends nowhere", "", true, "", "env"}, + {"blank sends nowhere", " ", true, "", "env"}, + {"plain http elsewhere", "http://submit.example.com", true, DefaultSubmitURL, "default"}, + {"path that is not absolute", "submit.json", true, DefaultSubmitURL, "default"}, + {"other scheme", "ftp://submit.example.com", true, DefaultSubmitURL, "default"}, + } + for _, c := range cases { + lookup := envOf(nil) + if c.set { + lookup = envOf(map[string]string{"CODEAF_MODEL_POOL_SUBMIT_URL": c.value}) + } + got := Resolve("", "", lookup) + if got.SubmitURL != c.want || got.Source.SubmitURL != c.wantSrc { + t.Errorf("%s: SubmitURL = %q from %q, want %q from %q", c.name, got.SubmitURL, got.Source.SubmitURL, c.want, c.wantSrc) + } + } +} + +func TestResolveRelayURL(t *testing.T) { + cases := []struct { + name string + value string + set bool + want string + wantSrc string + }{ + {"nobody set", "", false, DefaultRelayURL, "default"}, + {"https", "https://relay.example.com", true, "https://relay.example.com", "env"}, + {"https with a port", "https://relay.example.com:8443", true, "https://relay.example.com:8443", "env"}, + {"padded", " https://relay.example.com ", true, "https://relay.example.com", "env"}, + // A base is held with no trailing slash, whatever the name carried: the + // paths are appended to it, and a kept slash would make two. + {"trailing slash", "https://relay.example.com/", true, "https://relay.example.com", "env"}, + {"plain http elsewhere", "http://relay.example.com", true, DefaultRelayURL, "default"}, + {"other scheme", "ftp://relay.example.com", true, DefaultRelayURL, "default"}, + {"bare host", "relay.example.com", true, DefaultRelayURL, "default"}, + {"set and empty", "", true, DefaultRelayURL, "default"}, + } + for _, c := range cases { + lookup := envOf(nil) + if c.set { + lookup = envOf(map[string]string{"CODEAF_MODEL_POOL_RELAY_URL": c.value}) + } + got := Resolve("", "", lookup) + if got.RelayURL != c.want || got.Source.RelayURL != c.wantSrc { + t.Errorf("%s: RelayURL = %q from %q, want %q from %q", c.name, got.RelayURL, got.Source.RelayURL, c.want, c.wantSrc) + } + } +} + +func TestResolveMirrorURL(t *testing.T) { + cases := []struct { + name string + value string + set bool + want string + wantSrc string + }{ + {"nobody set", "", false, DefaultMirrorURL, "default"}, + {"https", "https://mirror.example.com/index.json", true, "https://mirror.example.com/index.json", "env"}, + {"http loopback with port", "http://127.0.0.1:9000/index.json", true, "http://127.0.0.1:9000/index.json", "env"}, + {"file", "file:///var/tmp/index.json", true, "file:///var/tmp/index.json", "env"}, + {"absolute path", "/var/tmp/index.json", true, "/var/tmp/index.json", "env"}, + {"padded", " https://mirror.example.com ", true, "https://mirror.example.com", "env"}, + // An empty mirror is an answer: it turns the fallback off. + {"empty disables the mirror", "", true, "", "env"}, + {"blank disables the mirror", " ", true, "", "env"}, + {"plain http elsewhere", "http://mirror.example.com/index.json", true, DefaultMirrorURL, "default"}, + {"other scheme", "ftp://mirror.example.com", true, DefaultMirrorURL, "default"}, + {"bare host", "mirror.example.com/index.json", true, DefaultMirrorURL, "default"}, + } + for _, c := range cases { + lookup := envOf(nil) + if c.set { + lookup = envOf(map[string]string{"CODEAF_MODEL_POOL_MIRROR_URL": c.value}) + } + got := Resolve("", "", lookup) + if got.MirrorURL != c.want || got.Source.MirrorURL != c.wantSrc { + t.Errorf("%s: MirrorURL = %q from %q, want %q from %q", c.name, got.MirrorURL, got.Source.MirrorURL, c.want, c.wantSrc) + } + } + + // The mirror is its own address: a pinned relay does not move it. + got := Resolve("", "", envOf(map[string]string{"CODEAF_MODEL_POOL_RELAY_URL": "https://relay.example.com"})) + if got.MirrorURL != DefaultMirrorURL || got.Source.MirrorURL != "default" { + t.Errorf("a pinned relay moved the mirror: %q from %q", got.MirrorURL, got.Source.MirrorURL) + } +} + +// The two addresses derive from the relay, and each override wins over its +// derivation. A relay read from the environment hands its source down to both +// derived addresses; an override of its own names itself instead. +func TestResolveDerivesTheAddressesFromTheRelay(t *testing.T) { + got := Resolve("", "", envOf(nil)) + if got.IndexURL != "https://codeaf.agentfield.ai/pool/index.json" || got.Source.IndexURL != "default" { + t.Errorf("IndexURL = %q from %q, want the relay's derived address from %q", got.IndexURL, got.Source.IndexURL, "default") + } + if got.SubmitURL != "https://codeaf.agentfield.ai/pool/v1/rows" || got.Source.SubmitURL != "default" { + t.Errorf("SubmitURL = %q from %q, want the relay's derived address from %q", got.SubmitURL, got.Source.SubmitURL, "default") + } + + lookup := envOf(map[string]string{"CODEAF_MODEL_POOL_RELAY_URL": "https://relay.example.com"}) + got = Resolve("", "", lookup) + if got.IndexURL != "https://relay.example.com/index.json" || got.Source.IndexURL != "env" { + t.Errorf("a pinned relay's index = %q from %q", got.IndexURL, got.Source.IndexURL) + } + if got.SubmitURL != "https://relay.example.com/v1/rows" || got.Source.SubmitURL != "env" { + t.Errorf("a pinned relay's submit = %q from %q", got.SubmitURL, got.Source.SubmitURL) + } + + // Each override still wins over its derivation, and the address it does + // not name keeps the relay's source. + lookup = envOf(map[string]string{ + "CODEAF_MODEL_POOL_RELAY_URL": "https://relay.example.com", + "CODEAF_MODEL_POOL_SUBMIT_URL": "https://submit.example.com", + }) + got = Resolve("", "", lookup) + if got.IndexURL != "https://relay.example.com/index.json" || got.Source.IndexURL != "env" { + t.Errorf("IndexURL = %q from %q", got.IndexURL, got.Source.IndexURL) + } + if got.SubmitURL != "https://submit.example.com" || got.Source.SubmitURL != "env" { + t.Errorf("SubmitURL = %q from %q", got.SubmitURL, got.Source.SubmitURL) + } + if got.RelayURL != "https://relay.example.com" { + t.Errorf("RelayURL = %q", got.RelayURL) + } +} + +func TestResolvePublicKey(t *testing.T) { + cases := []struct { + name string + setting string + value string + set bool + want string + wantSrc string + }{ + {"nobody set", "", "", false, "", "default"}, + {"env", "", "WOAo+g/oKxAV9vVqv2Q14w1TyyyiouwFO2fC0zgcps0=", true, "WOAo+g/oKxAV9vVqv2Q14w1TyyyiouwFO2fC0zgcps0=", "env"}, + {"env padded", "", " abc= ", true, "abc=", "env"}, + {"env set and empty falls through", "stored", "", true, "stored", "setting"}, + {"setting", "stored", "", false, "stored", "setting"}, + {"setting padded", " stored ", "", false, "stored", "setting"}, + {"env beats setting", "stored", "pinned", true, "pinned", "env"}, + } + for _, c := range cases { + lookup := envOf(nil) + if c.set { + lookup = envOf(map[string]string{"CODEAF_MODEL_POOL_PUBLIC_KEY": c.value}) + } + got := Resolve("", c.setting, lookup) + if got.PublicKey != c.want || got.Source.PublicKey != c.wantSrc { + t.Errorf("%s: PublicKey = %q from %q, want %q from %q", c.name, got.PublicKey, got.Source.PublicKey, c.want, c.wantSrc) + } + } + // The setting is Resolve's own argument, not a row the package reads: a + // nil lookup still carries one. + if got := Resolve("on", "stored", nil); got.PublicKey != "stored" || got.Source.PublicKey != "setting" { + t.Errorf("Resolve with a nil lookup lost the stored key: %+v", got) + } +} + +func TestResolveTTL(t *testing.T) { + cases := []struct { + name string + value string + set bool + want time.Duration + wantSrc string + }{ + {"nobody set", "", false, 24 * time.Hour, "default"}, + {"readable", "2h45m", true, 2*time.Hour + 45*time.Minute, "env"}, + {"padded", " 3h ", true, 3 * time.Hour, "env"}, + {"under the floor", "30s", true, time.Minute, "env"}, + {"at the floor", "1m", true, time.Minute, "env"}, + {"negative", "-1h", true, time.Minute, "env"}, + {"zero", "0", true, time.Minute, "env"}, + {"over the ceiling", "1000h", true, 7 * 24 * time.Hour, "env"}, + {"at the ceiling", "168h", true, 7 * 24 * time.Hour, "env"}, + {"unreadable", "banana", true, 24 * time.Hour, "default"}, + {"no day unit", "7d", true, 24 * time.Hour, "default"}, + {"set and empty", "", true, 24 * time.Hour, "default"}, + } + for _, c := range cases { + lookup := envOf(nil) + if c.set { + lookup = envOf(map[string]string{"CODEAF_MODEL_POOL_TTL": c.value}) + } + got := Resolve("", "", lookup) + if got.TTL != c.want || got.Source.TTL != c.wantSrc { + t.Errorf("%s: TTL = %v from %q, want %v from %q", c.name, got.TTL, got.Source.TTL, c.want, c.wantSrc) + } + } +} + +func TestResolveFieldsAreIndependent(t *testing.T) { + // Off, an unreadable URL and a readable TTL: the mode does not change how + // the URLs or the TTL are read, and the unreadable URL does not disturb + // the TTL. + got := Resolve("bogus", "", envOf(map[string]string{ + "CODEAF_MODEL_POOL": "off", + "CODEAF_MODEL_POOL_URL": "http://pool.example.com", + "CODEAF_MODEL_POOL_SUBMIT_URL": "https://submit.example.com", + "CODEAF_MODEL_POOL_TTL": "2h", + })) + want := Config{ + Mode: Off, + RelayURL: DefaultRelayURL, + IndexURL: DefaultIndexURL, + SubmitURL: "https://submit.example.com", + MirrorURL: DefaultMirrorURL, + PublicKey: "", + TTL: 2 * time.Hour, + Source: Sources{ + Mode: "env", + RelayURL: "default", + IndexURL: "default", + SubmitURL: "env", + MirrorURL: "default", + PublicKey: "default", + TTL: "env", + }, + } + if got != want { + t.Errorf("Resolve() = %+v, want %+v", got, want) + } + + // An unreadable TTL does not disturb the URLs either. + got = Resolve("", "", envOf(map[string]string{ + "CODEAF_MODEL_POOL_URL": "https://pool.example.com", + "CODEAF_MODEL_POOL_TTL": "banana", + })) + if got.IndexURL != "https://pool.example.com" || got.Source.IndexURL != "env" { + t.Errorf("IndexURL = %q from %q, want the env value from %q", got.IndexURL, got.Source.IndexURL, "env") + } + if got.TTL != 24*time.Hour || got.Source.TTL != "default" { + t.Errorf("TTL = %v from %q, want 24h from %q", got.TTL, got.Source.TTL, "default") + } +} + +func TestCanSendAndCanRead(t *testing.T) { + cases := []struct { + name string + config Config + wantCanSend bool + wantCanRead bool + }{ + {"on with an address", Config{Mode: On, SubmitURL: DefaultSubmitURL}, true, true}, + {"on with no address", Config{Mode: On}, false, true}, + {"read with an address", Config{Mode: Read, SubmitURL: DefaultSubmitURL}, false, true}, + {"read with no address", Config{Mode: Read}, false, true}, + {"off with an address", Config{Mode: Off, SubmitURL: DefaultSubmitURL}, false, false}, + } + for _, c := range cases { + if got := c.config.CanSend(); got != c.wantCanSend { + t.Errorf("%s: CanSend() = %v, want %v", c.name, got, c.wantCanSend) + } + if got := c.config.CanRead(); got != c.wantCanRead { + t.Errorf("%s: CanRead() = %v, want %v", c.name, got, c.wantCanRead) + } + } +} + +func TestResolvedConfigCanSendAndCanRead(t *testing.T) { + // An empty submit address turns sending off while everything else stays on. + got := Resolve("", "", envOf(map[string]string{"CODEAF_MODEL_POOL_SUBMIT_URL": ""})) + if got.SubmitURL != "" || got.Source.SubmitURL != "env" { + t.Fatalf("SubmitURL = %q from %q, want %q from %q", got.SubmitURL, got.Source.SubmitURL, "", "env") + } + if got.CanSend() { + t.Error("an empty submit address must not send") + } + if !got.CanRead() { + t.Error("an empty submit address must still read") + } + + if got := Resolve("", "", envOf(nil)); !got.CanSend() || !got.CanRead() { + t.Errorf("the defaults must send and read: CanSend() = %v, CanRead() = %v", got.CanSend(), got.CanRead()) + } + if got := Resolve("read", "", envOf(nil)); got.CanSend() || !got.CanRead() { + t.Errorf("read must read and not send: CanSend() = %v, CanRead() = %v", got.CanSend(), got.CanRead()) + } + if got := Resolve("off", "", envOf(nil)); got.CanSend() || got.CanRead() { + t.Errorf("off must neither send nor read: CanSend() = %v, CanRead() = %v", got.CanSend(), got.CanRead()) + } +} + +func TestResolveNilLookupUsesSetting(t *testing.T) { + got := Resolve("read", "", nil) + if got.Mode != Read || got.Source.Mode != "setting" { + t.Errorf("Resolve(\"read\", \"\", nil): Mode = %v from %q, want read from %q", got.Mode, got.Source.Mode, "setting") + } + if got.IndexURL != DefaultIndexURL || got.SubmitURL != DefaultSubmitURL || got.RelayURL != DefaultRelayURL || got.MirrorURL != DefaultMirrorURL || got.TTL != 24*time.Hour { + t.Errorf("Resolve(\"read\", \"\", nil) = %+v, want the defaults beside the setting", got) + } +} + +func TestResolveAsksEightNames(t *testing.T) { + var asked []string + Resolve("", "", func(name string) (string, bool) { + asked = append(asked, name) + return "", false + }) + sort.Strings(asked) + want := []string{envMode, envCI, envRelay, envIndex, envSubmit, envMirror, envTTL, envKey} + sort.Strings(want) + if !reflect.DeepEqual(asked, want) { + t.Errorf("Resolve asked lookup for %v, want %v", asked, want) + } +} + +func TestResolveIsPure(t *testing.T) { + first := envOf(map[string]string{"CODEAF_MODEL_POOL": "read", "CODEAF_MODEL_POOL_TTL": "3h"}) + second := envOf(map[string]string{"CODEAF_MODEL_POOL": "off"}) + want := Resolve("", "", first) + Resolve("", "", second) + if got := Resolve("", "", first); got != want { + t.Errorf("a second call kept state: %+v, want %+v", got, want) + } + if got := Resolve("", "", second); got.Mode != Off || got.TTL != 24*time.Hour { + t.Errorf("a fresh environment was polluted: %+v", got) + } +} + +func TestResolveConcurrent(t *testing.T) { + lookup := envOf(map[string]string{ + "CODEAF_MODEL_POOL": "read", + "CODEAF_MODEL_POOL_TTL": "90m", + }) + want := Resolve("", "", lookup) + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 64; j++ { + if got := Resolve("", "", lookup); got != want { + t.Errorf("Resolve() = %+v, want %+v", got, want) + return + } + } + }() + } + wg.Wait() +} diff --git a/internal/pool/pull/cache.go b/internal/pool/pull/cache.go new file mode 100644 index 0000000000..040ca559cf --- /dev/null +++ b/internal/pool/pull/cache.go @@ -0,0 +1,136 @@ +package pull + +import ( + "encoding/json" + "os" + "path/filepath" + "time" +) + +// cache is the on-disk record of the last good fetch: the document, its +// signature, its ETag and when it was fetched. It is written so that a run cut +// in half never leaves a half-written cache: each file goes to a temporary name +// in CacheDir and is then renamed into place, and the files it leaves behind +// have mode 0600. +// +// A cache that does not read is no cache. Bytes that do not parse, or whose +// signature no longer verifies under [Puller.Keys], are treated as if nothing were +// cached: no error of their own, no panic, and the next good fetch replaces +// them. +type cache struct { + doc []byte + sig []byte + etag string + version int64 + fetchedAt time.Time + ok bool +} + +// loadCache reads the cache directory and returns what it finds, or a cache +// with ok false. A cache that fails to read is not an error: it is treated as +// no cache at all. +func (p *Puller) loadCache() (cache, bool) { + var c cache + doc, err := readLimitedFile(filepath.Join(p.CacheDir, "doc.json")) + if err != nil { + return c, false + } + sig, err := readLimitedFile(filepath.Join(p.CacheDir, "doc.sig")) + if err != nil { + return c, false + } + meta, err := os.ReadFile(filepath.Join(p.CacheDir, "meta.json")) + if err != nil { + return c, false + } + var m struct { + ETag string `json:"etag"` + Version int64 `json:"version"` + FetchedAt time.Time `json:"fetched_at"` + } + if err := json.Unmarshal(meta, &m); err != nil { + return c, false + } + // THE CACHE IS NOT A TRUSTED STORE. A signature that no longer verifies + // under [Puller.Keys] means the cache is stale, not that the caller is owed an + // error. + if !Verify(doc, sig, p.Keys) { + return c, false + } + // AND THE VERSION COMES OFF THE DOCUMENT, not off the file beside it. The + // three files are committed by three renames, so a run cut between them + // leaves a doc and a signature that verify as a matching pair while + // meta.json still describes the copy before. Trusting that number would + // hand a caller one document's bytes under another's version, and — worse + // — would check the next fetch's version against the older one, which is + // the rollback the version floor exists to catch. A cache whose metadata + // does not describe the bytes it sits beside is a cache that does not read. + version, err := docVersion(doc) + if err != nil || version != m.Version { + return c, false + } + c.doc = doc + c.sig = sig + c.etag = m.ETag + c.version = version + c.fetchedAt = m.FetchedAt + c.ok = true + return c, true +} + +// storeCache writes the document, its signature, its ETag and when it was +// fetched to CacheDir. Each file goes to a temporary name and is then renamed +// into place, so a run cut in half never leaves a half-written cache. The +// files it leaves behind have mode 0600. CacheDir is made if it is missing. +func (p *Puller) storeCache(doc, sig []byte, etag string, version int64, fetchedAt time.Time) { + if err := os.MkdirAll(p.CacheDir, 0o700); err != nil { + // A cache that cannot be written is not fatal: the caller still gets + // its document, and the next run simply fetches again. + return + } + meta, err := json.Marshal(struct { + ETag string `json:"etag"` + Version int64 `json:"version"` + FetchedAt time.Time `json:"fetched_at"` + }{etag, version, fetchedAt}) + if err != nil { + return + } + files := []struct { + name string + data []byte + }{ + {"doc.json", doc}, + {"doc.sig", sig}, + {"meta.json", meta}, + } + for _, f := range files { + if err := atomicWrite(p.CacheDir, f.name, f.data); err != nil { + return + } + } +} + +// atomicWrite writes data to a temporary file in dir and renames it into +// place. The file is created with mode 0600, so the files the cache leaves +// behind are owner-readable only. +func atomicWrite(dir, name string, data []byte) error { + tmp, err := os.CreateTemp(dir, "."+name+".*") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) // a rename that succeeded leaves no temp behind + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpPath, filepath.Join(dir, name)) +} diff --git a/internal/pool/pull/pull.go b/internal/pool/pull/pull.go new file mode 100644 index 0000000000..c2c190e434 --- /dev/null +++ b/internal/pool/pull/pull.go @@ -0,0 +1,431 @@ +// Package pull fetches a signed JSON document, checks its signature against +// keys it was handed, and keeps the last good copy on disk so that the next run +// has something to work with when the fetch does not go through. +// +// THE SHAPE IT EXISTS FOR. A document this package serves changes rarely and is +// needed on every run; the source that publishes it is not always reachable. +// The whole point is that a caller can ask for the document on any run and get +// an answer: either a fresh one, or the last good one with an error saying why +// it is not fresh. The network is consulted only when the cache is older than +// the caller's patience, and every failure the source can throw — a dead +// host, a timeout, a bad signature, a rollback — falls back to the cache +// rather than leaving the caller empty. +// +// WHAT MAKES A DOCUMENT GOOD is written down in [Puller.good] and nowhere else, +// because it is the one contract a reader can spot on its own: the document is +// JSON, an object, and carries an integer "version"; its ".sig" verifies under +// one of the keys the caller trusts; and its version is not lower than the one +// the cache already holds, because a copy of yesterday served today is the one +// attack a reader can catch without trusting the source. A good document is +// cached and returned; a bad one returns whatever the cache had, together with +// the error that says why the fresh one was refused. +// +// THE CACHE IS NOT A TRUSTED STORE. Bytes in [Puller.CacheDir] that do not +// parse, or whose signature no longer verifies under [Puller.Keys], are treated +// as if nothing were cached: no error of their own, no panic, and the next good +// fetch replaces them. The cache is a copy of a good fetch, never a source of +// truth in its own right. +package pull + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" +) + +// errNotModified is the sentinel a 304 answer returns. It is not a failure: +// the cached copy is the answer, and Pull handles it before the fallback path. +var errNotModified = errors.New("pull: not modified") + +// ErrBadSignature is a fetched document whose signature does not verify under +// any of the keys in hand. It is its own error because a caller deciding +// whether to look elsewhere for the same document has to tell a document that +// failed its check from a source that did not answer: a copy somewhere else +// cannot vouch for bytes whose signature just failed. +var ErrBadSignature = errors.New("pull: signature does not verify") + +const ( + // MaxDoc is the largest document Pull will read. A document larger than + // this is refused rather than read to the end, because a source that answers + // with an unbounded body is a source that can exhaust the reader. + MaxDoc = 4 << 20 // 4 MiB + + // DefaultBudget is what a Pull with a zero Budget spends. It is short on + // purpose: this package serves a document that is needed on the path of a + // keystroke, and a caller who can afford to wait longer sets its own. + DefaultBudget = 2 * time.Second +) + +// Result is what a single Pull returns. Doc is the document bytes, nil when +// there is nothing to give. Version is its "version". FromCache is true when +// the bytes came off disk, not off the wire. Changed is true when this version +// is newer than the one the cache held. +type Result struct { + Doc []byte + Version int64 + FromCache bool + Changed bool +} + +// Puller fetches one document from one URL, verifies it against one set of +// keys, and keeps the last good copy in one directory. The zero value is not +// usable: URL, Keys and CacheDir must be set before Pull is called. +// +// Client and Now have nil defaults the package fills: a plain http.Client with +// no timeout of its own (the budget governs) and time.Now. A caller that wants +// deterministic clocks or a test transport sets both. +type Puller struct { + URL string + Keys []ed25519.PublicKey + CacheDir string + TTL time.Duration + Budget time.Duration // 0 means DefaultBudget + Client *http.Client // nil means a client the package makes + Now func() time.Time // nil means time.Now +} + +// Verify answers whether sig is a detached ed25519 signature over doc under ANY +// ONE of keys. sig is read as base64 text in the standard encoding; space +// around it is ignored. No keys, an unreadable base64, a signature of the +// wrong length or a signature nobody's key made is false. A caller who trusts +// nobody trusts nothing. +// +// The key-length guard is here because ed25519.Verify panics on a wrong-length +// key, and a key the caller mis-hands is a misconfiguration, not a crash. +func Verify(doc, sig []byte, keys []ed25519.PublicKey) bool { + trimmed := bytes.TrimSpace(sig) + raw, err := base64.StdEncoding.DecodeString(string(trimmed)) + if err != nil { + return false + } + if len(raw) != ed25519.SignatureSize { + return false + } + for _, k := range keys { + if len(k) != ed25519.PublicKeySize { + continue + } + if ed25519.Verify(k, doc, raw) { + return true + } + } + return false +} + +// Pull fetches the document and returns a Result and an error. It never +// panics. The whole call lives inside Budget (DefaultBudget when it is zero) +// and inside ctx, whichever ends first. +func (p *Puller) Pull(ctx context.Context) (Result, error) { + if p.URL == "" { + return Result{}, errors.New("pull: empty url") + } + budget := p.Budget + if budget <= 0 { + budget = DefaultBudget + } + ctx, cancel := context.WithTimeout(ctx, budget) + defer cancel() + + now := p.now() + cached, ok := p.loadCache() + + // THE NETWORK IS SKIPPED WHEN THE CACHE IS YOUNG. A good copy fetched less + // than TTL ago answers straight from the cache, and not one request is + // made. A TTL of zero makes no cached copy young. + if ok && p.TTL > 0 && now.Sub(cached.fetchedAt) < p.TTL { + return Result{ + Doc: cached.doc, + Version: cached.version, + FromCache: true, + Changed: false, + }, nil + } + + doc, sig, etag, err := p.fetch(ctx, cached) + if err != nil { + // A 304 is not a failure: the cached copy is the answer. But a 304 with + // no cache is a source claiming not-modified for something we never + // had, which is a failure. + if errors.Is(err, errNotModified) && ok { + return Result{ + Doc: cached.doc, + Version: cached.version, + FromCache: true, + Changed: false, + }, nil + } + // EVERY FAILURE FALLS BACK. The cached good copy, if there is one, is + // returned with the error that says why the fresh one was refused. The + // cache is left exactly as it was. + if ok { + return Result{ + Doc: cached.doc, + Version: cached.version, + FromCache: true, + Changed: false, + }, err + } + return Result{FromCache: false}, err + } + + // WHAT MAKES A FETCHED DOCUMENT GOOD. + version, err := p.good(doc, sig, cached) + if err != nil { + if ok { + return Result{ + Doc: cached.doc, + Version: cached.version, + FromCache: true, + Changed: false, + }, err + } + return Result{FromCache: false}, err + } + + changed := true + if ok && version == cached.version { + changed = false + } + + // A good document is cached and returned with FromCache false. + p.storeCache(doc, sig, etag, version, now) + return Result{ + Doc: doc, + Version: version, + FromCache: false, + Changed: changed, + }, nil +} + +// good answers whether the fetched bytes are a document this package will serve. +// The document must be JSON, an object, and carry an integer "version"; its +// signature must verify under one of [Puller.Keys]; and its version may not be lower +// than the one the cache already holds. It returns the version when it does. +func (p *Puller) good(doc, sig []byte, cached cache) (int64, error) { + if !Verify(doc, sig, p.Keys) { + return 0, ErrBadSignature + } + version, err := docVersion(doc) + if err != nil { + return 0, err + } + if cached.ok && version < cached.version { + return 0, fmt.Errorf("pull: version %d is lower than cached %d", version, cached.version) + } + return version, nil +} + +// docVersion reads the "version" a document carries. It is the half of [good] +// that is about the bytes alone, split out because the cache reads it too: the +// version a cached copy is trusted for has to come from the bytes whose +// signature was just checked, not from a file beside them. +func docVersion(doc []byte) (int64, error) { + var obj map[string]json.RawMessage + if err := json.Unmarshal(doc, &obj); err != nil { + return 0, fmt.Errorf("pull: document is not json: %w", err) + } + if obj == nil { + return 0, errors.New("pull: document is not a json object") + } + raw, has := obj["version"] + if !has { + return 0, errors.New("pull: document has no version") + } + version, err := strconv.ParseInt(string(raw), 10, 64) + if err != nil { + return 0, fmt.Errorf("pull: version is not an integer: %w", err) + } + return version, nil +} + +func (p *Puller) now() time.Time { + if p.Now != nil { + return p.Now() + } + return time.Now() +} + +func (p *Puller) client() *http.Client { + if p.Client != nil { + return p.Client + } + return &http.Client{} +} + +// sigLocation names the signature location: the URL or path with ".sig" appended to +// its text. A URL gets ".sig" on the end of the URL; a path gets ".sig" on the +// end of the path. +func (p *Puller) sigLocation() string { + return p.URL + ".sig" +} + +// fetch reads the document and its signature from the source, and returns the +// document bytes, the raw signature bytes, the ETag (if any) and any error. +// Over http, a cached copy the source may still have unchanged is asked for +// with If-None-Match carrying the ETag that came with it; a 304 means the +// cached copy is the answer. +func (p *Puller) fetch(ctx context.Context, cached cache) ([]byte, []byte, string, error) { + scheme := schemeOf(p.URL) + switch scheme { + case "http", "https": + return p.fetchHTTP(ctx, cached) + default: + return p.fetchFile(ctx) + } +} + +// fetchHTTP reads the document and its signature over the network. +func (p *Puller) fetchHTTP(ctx context.Context, cached cache) ([]byte, []byte, string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil) + if err != nil { + return nil, nil, "", fmt.Errorf("pull: build request: %w", err) + } + if cached.ok && cached.etag != "" { + req.Header.Set("If-None-Match", cached.etag) + } + resp, err := p.client().Do(req) + if err != nil { + return nil, nil, "", fmt.Errorf("pull: fetch: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotModified { + // A 304 means the cached copy is still the source's current one. The + // caller already has those bytes; the sentinel tells Pull to answer + // from the cache with a nil error. + return nil, nil, "", errNotModified + } + if resp.StatusCode != http.StatusOK { + return nil, nil, "", fmt.Errorf("pull: fetch: status %d", resp.StatusCode) + } + doc, err := readLimited(resp.Body, resp.ContentLength) + if err != nil { + return nil, nil, "", fmt.Errorf("pull: read document: %w", err) + } + sig, err := p.fetchSigHTTP(ctx) + if err != nil { + return nil, nil, "", err + } + return doc, sig, resp.Header.Get("ETag"), nil +} + +func (p *Puller) fetchSigHTTP(ctx context.Context) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.sigLocation(), nil) + if err != nil { + return nil, fmt.Errorf("pull: build signature request: %w", err) + } + resp, err := p.client().Do(req) + if err != nil { + return nil, fmt.Errorf("pull: fetch signature: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("pull: fetch signature: status %d", resp.StatusCode) + } + sig, err := readLimited(resp.Body, resp.ContentLength) + if err != nil { + return nil, fmt.Errorf("pull: read signature: %w", err) + } + return sig, nil +} + +// fetchFile reads the document and its signature from the filesystem. A path +// may be written plain or as "file://" and then the path. +// The budget and ctx bound this read as they bound the network one. A local +// file is served in microseconds, so the deadline is not there to cut a slow +// read off; it is there so that Pull means the same thing whatever the URL +// names — a caller that handed in a cancelled context is answered with that +// cancellation and not with a fresh document. +func (p *Puller) fetchFile(ctx context.Context) ([]byte, []byte, string, error) { + if err := ctx.Err(); err != nil { + return nil, nil, "", fmt.Errorf("pull: read document: %w", err) + } + path := stripFile(p.URL) + doc, err := readLimitedFile(path) + if err != nil { + return nil, nil, "", fmt.Errorf("pull: read document: %w", err) + } + if err := ctx.Err(); err != nil { + return nil, nil, "", fmt.Errorf("pull: read signature: %w", err) + } + sig, err := readLimitedFile(stripFile(p.sigLocation())) + if err != nil { + return nil, nil, "", fmt.Errorf("pull: read signature: %w", err) + } + return doc, sig, "", nil +} + +// readLimited reads from r up to MaxDoc bytes. A ContentLength above MaxDoc is +// refused before a byte is read; a body that exceeds MaxDoc mid-stream is cut +// off and refused rather than read to the end. +func readLimited(r io.Reader, contentLength int64) ([]byte, error) { + if contentLength > MaxDoc { + return nil, fmt.Errorf("size %d exceeds max %d", contentLength, MaxDoc) + } + // MaxDoc+1 so a body that fills the buffer once is caught as too large. + limited := io.LimitReader(r, MaxDoc+1) + data, err := io.ReadAll(limited) + if err != nil { + return nil, err + } + if len(data) > MaxDoc { + return nil, fmt.Errorf("size exceeds max %d", MaxDoc) + } + return data, nil +} + +// readLimitedFile reads a file up to MaxDoc bytes. A file larger than MaxDoc is +// refused before it is read to the end. +func readLimitedFile(path string) ([]byte, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.Size() > MaxDoc { + return nil, fmt.Errorf("size %d exceeds max %d", info.Size(), MaxDoc) + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + limited := io.LimitReader(f, MaxDoc+1) + data, err := io.ReadAll(limited) + if err != nil { + return nil, err + } + if len(data) > MaxDoc { + return nil, fmt.Errorf("size exceeds max %d", MaxDoc) + } + return data, nil +} + +// schemeOf returns the scheme of a URL, or "" when the string is a plain +// filesystem path. A "file://" prefix is reported as "file". +func schemeOf(s string) string { + if strings.HasPrefix(s, "file://") { + return "file" + } + if u, err := url.Parse(s); err == nil && u.Scheme != "" { + return u.Scheme + } + return "" +} + +// stripFile removes a "file://" prefix, leaving the bare path. +func stripFile(s string) string { + if strings.HasPrefix(s, "file://") { + return strings.TrimPrefix(s, "file://") + } + return s +} diff --git a/internal/pool/pull/pull_test.go b/internal/pool/pull/pull_test.go new file mode 100644 index 0000000000..018ba073d0 --- /dev/null +++ b/internal/pool/pull/pull_test.go @@ -0,0 +1,1214 @@ +package pull + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// makeKey makes a fresh ed25519 key pair. +func makeKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return pub, priv +} + +// signDoc signs doc with priv and returns base64-encoded standard encoding. +func signDoc(t *testing.T, priv ed25519.PrivateKey, doc []byte) []byte { + t.Helper() + sig := ed25519.Sign(priv, doc) + return []byte(base64.StdEncoding.EncodeToString(sig)) +} + +// jsonObject builds a JSON object with a version field and whatever extra +// fields are given. +func jsonObject(version int64, extra map[string]any) []byte { + obj := map[string]any{"version": version} + for k, v := range extra { + obj[k] = v + } + b, _ := json.Marshal(obj) + return b +} + +// writeFile is the test shorthand that makes a 0600 file. +func writeFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} + +// ----------------------------------------------------------------------------- + +// Verify reads a base64 standard-encoding signature and answers whether it +// is a detached ed25519 signature over doc under any one of the keys. No keys, +// unreadable base64, the wrong length, or a signature nobody's key made is +// false. +func TestVerifyAnswersTrueForAnyTrustedKey(t *testing.T) { + pub1, priv1 := makeKey(t) + pub2, priv2 := makeKey(t) + doc := []byte(`{"version":3}`) + sig1 := signDoc(t, priv1, doc) + sig2 := signDoc(t, priv2, doc) + + if !Verify(doc, sig1, []ed25519.PublicKey{pub1, pub2}) { + t.Fatal("signature by key 1 did not verify") + } + if !Verify(doc, sig2, []ed25519.PublicKey{pub1, pub2}) { + t.Fatal("signature by key 2 did not verify") + } +} + +// A caller who trusts nobody trusts nothing. +func TestVerifyWithNoKeysIsFalse(t *testing.T) { + _, priv := makeKey(t) + doc := []byte(`{"version":1}`) + sig := signDoc(t, priv, doc) + if Verify(doc, sig, nil) { + t.Fatal("a signature verified with no keys") + } +} + +// Unreadable base64 is false, not an error. +func TestVerifyWithGarbageSignatureIsFalse(t *testing.T) { + pub, _ := makeKey(t) + doc := []byte(`{"version":1}`) + if Verify(doc, []byte("!!!not-base64!!!"), []ed25519.PublicKey{pub}) { + t.Fatal("garbage verified") + } +} + +// A signature of the wrong length is false. +func TestVerifyWithWrongLengthSignatureIsFalse(t *testing.T) { + pub, _ := makeKey(t) + doc := []byte(`{"version":1}`) + short := base64.StdEncoding.EncodeToString([]byte("too short")) + if Verify(doc, []byte(short), []ed25519.PublicKey{pub}) { + t.Fatal("a short signature verified") + } +} + +// Space around the signature is ignored. +func TestVerifyIgnoresSpaceAroundSignature(t *testing.T) { + pub, priv := makeKey(t) + doc := []byte(`{"version":1}`) + sig := signDoc(t, priv, doc) + padded := []byte(" \n" + string(sig) + "\t ") + if !Verify(doc, padded, []ed25519.PublicKey{pub}) { + t.Fatal("padded signature did not verify") + } +} + +// A signature over the wrong bytes is false. +func TestVerifyWithWrongMessageIsFalse(t *testing.T) { + pub, priv := makeKey(t) + sig := signDoc(t, priv, []byte(`{"version":1}`)) + if Verify([]byte(`{"version":2}`), sig, []ed25519.PublicKey{pub}) { + t.Fatal("a signature verified over the wrong message") + } +} + +// A wrong-length key is skipped rather than crashing the caller. +func TestVerifySkipsWrongLengthKeys(t *testing.T) { + pub, priv := makeKey(t) + doc := []byte(`{"version":1}`) + sig := signDoc(t, priv, doc) + bad := ed25519.PublicKey(make([]byte, 10)) // not PublicKeySize + if !Verify(doc, sig, []ed25519.PublicKey{bad, pub}) { + t.Fatal("a good key behind a bad one did not verify") + } +} + +// ----------------------------------------------------------------------------- + +// A good file-source fetch returns the document with FromCache false and +// Changed true when nothing was cached. +func TestPullFromFileReturnsFreshDocument(t *testing.T) { + pub, priv := makeKey(t) + dir := t.TempDir() + doc := jsonObject(5, nil) + sig := signDoc(t, priv, doc) + writeFile(t, filepath.Join(dir, "doc.json"), doc) + writeFile(t, filepath.Join(dir, "doc.json.sig"), sig) + + p := &Puller{ + URL: filepath.Join(dir, "doc.json"), + Keys: []ed25519.PublicKey{pub}, + CacheDir: t.TempDir(), + } + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("Pull failed: %v", err) + } + if res.FromCache { + t.Fatal("a fresh fetch reported FromCache") + } + if !res.Changed { + t.Fatal("a first fetch reported Changed false") + } + if res.Version != 5 { + t.Fatalf("Version = %d, want 5", res.Version) + } + if !bytesEqual(res.Doc, doc) { + t.Fatal("Doc does not match the source") + } +} + +// A file:// URL is the same fetch as a plain path. +func TestPullFromFileURLScheme(t *testing.T) { + pub, priv := makeKey(t) + dir := t.TempDir() + doc := jsonObject(1, nil) + sig := signDoc(t, priv, doc) + path := filepath.Join(dir, "doc.json") + writeFile(t, path, doc) + writeFile(t, path+".sig", sig) + + p := &Puller{ + URL: "file://" + path, + Keys: []ed25519.PublicKey{pub}, + CacheDir: t.TempDir(), + } + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("Pull failed: %v", err) + } + if res.Version != 1 { + t.Fatalf("Version = %d, want 1", res.Version) + } +} + +// ----------------------------------------------------------------------------- + +// A fetchServer serves a document and its signature and remembers how many +// times each was asked for. +type fetchServer struct { + t *testing.T + pub ed25519.PublicKey + priv ed25519.PrivateKey + docBody []byte + sigBody []byte + etag string + docHits int + sigHits int + statusDoc int + statusSig int + delay time.Duration + modified bool +} + +func newFetchServer(t *testing.T) (*fetchServer, *httptest.Server) { + t.Helper() + pub, priv := makeKey(t) + fs := &fetchServer{ + t: t, + pub: pub, + priv: priv, + docBody: jsonObject(1, nil), + statusDoc: http.StatusOK, + statusSig: http.StatusOK, + etag: "v1", + } + fs.sigBody = signDoc(t, priv, fs.docBody) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if fs.delay > 0 { + time.Sleep(fs.delay) + } + if strings.HasSuffix(r.URL.Path, ".sig") { + fs.sigHits++ + w.Header().Set("Content-Type", "text/plain") + if fs.statusSig != http.StatusOK { + w.WriteHeader(fs.statusSig) + return + } + _, _ = w.Write(fs.sigBody) + return + } + fs.docHits++ + if r.Header.Get("If-None-Match") == fs.etag && !fs.modified { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", fs.etag) + if fs.statusDoc != http.StatusOK { + w.WriteHeader(fs.statusDoc) + return + } + _, _ = w.Write(fs.docBody) + })) + t.Cleanup(srv.Close) + return fs, srv +} + +func (fs *fetchServer) setVersion(v int64) { + fs.docBody = jsonObject(v, nil) + fs.sigBody = signDoc(fs.t, fs.priv, fs.docBody) + fs.etag = "v" + strconv.FormatInt(v, 10) + fs.modified = true +} + +// ----------------------------------------------------------------------------- + +// A good http fetch returns the document fresh. +func TestPullOverHTTPReturnsFreshDocument(t *testing.T) { + fs, srv := newFetchServer(t) + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: t.TempDir(), + } + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("Pull failed: %v", err) + } + if res.FromCache { + t.Fatal("fresh fetch reported FromCache") + } + if !res.Changed { + t.Fatal("first fetch reported Changed false") + } + if fs.docHits != 1 { + t.Fatalf("doc requested %d times, want 1", fs.docHits) + } +} + +// THE NETWORK IS SKIPPED WHEN THE CACHE IS YOUNG. A good copy cached less than +// TTL ago answers from the cache, and not one request is made. +func TestCacheYoungSkipsNetworkEntirely(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + clock := time.Now() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 10 * time.Minute, + Now: func() time.Time { return clock }, + } + // First fetch hits the network and caches. + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + if fs.docHits != 1 { + t.Fatalf("first pull hit the doc %d times, want 1", fs.docHits) + } + // Second fetch is inside the TTL: no request at all. + clock = clock.Add(5 * time.Minute) + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("second pull failed: %v", err) + } + if !res.FromCache { + t.Fatal("a fetch inside TTL did not come from cache") + } + if res.Changed { + t.Fatal("a fetch inside TTL reported Changed") + } + if fs.docHits != 1 { + t.Fatalf("second pull hit the doc %d times, want 1", fs.docHits) + } +} + +// A TTL of zero makes no cached copy young: the source is always consulted. +func TestZeroTTLAlwaysConsultsSource(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + if fs.docHits != 2 { + t.Fatalf("two pulls with TTL 0 hit the doc %d times, want 2", fs.docHits) + } +} + +// ----------------------------------------------------------------------------- + +// Over http, a cached copy the source still has unchanged is asked for with +// If-None-Match; a 304 means the cached copy is the answer. +func TestHTTP304MeansCachedCopyIsAnswer(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, // force a network consult + } + // First fetch caches with an ETag. + first, err := p.Pull(t.Context()) + if err != nil { + t.Fatal(err) + } + if first.Changed != true { + t.Fatal("first fetch should be Changed") + } + // Second fetch: the source answers 304 for the same ETag. + fs.modified = false + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("second pull failed: %v", err) + } + if !res.FromCache { + t.Fatal("a 304 did not produce FromCache") + } + if res.Changed { + t.Fatal("a 304 reported Changed") + } + if fs.docHits != 2 { + t.Fatalf("doc requested %d times, want 2 (both the GET and the 304)", fs.docHits) + } +} + +// A higher version is Changed true; an equal version is Changed false. +func TestChangedFlagReflectsVersion(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if res, _ := p.Pull(t.Context()); !res.Changed { + t.Fatal("first fetch should be Changed") + } + // Same version again: Changed false, still a fetch (not 304, no etag match + // because modified flag default makes the server return 200). + fs.modified = false + if res, _ := p.Pull(t.Context()); res.Changed { + t.Fatal("same version reported Changed") + } + // Higher version: Changed true. + fs.setVersion(2) + fs.modified = true + res, _ := p.Pull(t.Context()) + if !res.Changed { + t.Fatal("higher version reported Changed false") + } + if res.Version != 2 { + t.Fatalf("Version = %d, want 2", res.Version) + } +} + +// ----------------------------------------------------------------------------- + +// A document larger than MaxDoc is refused rather than read to the end. +func TestDocumentLargerThanMaxDocIsRefused(t *testing.T) { + // Override MaxDoc locally by testing the reader directly, since the real + // MaxDoc is 4 MiB and writing 5 MiB of test data is wasteful. + data := make([]byte, MaxDoc+1) + for i := range data { + data[i] = 'a' + } + _, err := readLimited(strings.NewReader(string(data)), int64(len(data))) + if err == nil { + t.Fatal("a document larger than MaxDoc was read without error") + } + if !strings.Contains(err.Error(), "exceeds max") { + t.Fatalf("error does not name the size: %v", err) + } +} + +// ContentLength above MaxDoc is refused before a byte is read. +func TestContentLengthAboveMaxDocIsRefusedBeforeRead(t *testing.T) { + called := false + r := readerFunc{read: func(b []byte) (int, error) { + called = true + return 0, nil + }} + _, err := readLimited(r, int64(MaxDoc+1)) + if err == nil { + t.Fatal("a ContentLength above MaxDoc was not refused") + } + if called { + t.Fatal("the body was read despite ContentLength above MaxDoc") + } +} + +type readerFunc struct { + read func([]byte) (int, error) +} + +func (r readerFunc) Read(b []byte) (int, error) { return r.read(b) } + +// A document too large over http falls back to the cache. +func TestTooLargeOverHTTPFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + // Serve a document that is one byte too large. + big := make([]byte, MaxDoc+1) + for i := range big { + big[i] = 'a' + } + fs.docBody = big + fs.sigBody = signDoc(t, fs.priv, big) + fs.modified = true + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a too-large document returned no error") + } + if !res.FromCache { + t.Fatal("a too-large document did not fall back to cache") + } +} + +// ----------------------------------------------------------------------------- + +// A bad signature falls back to the cache. +func TestBadSignatureFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + // First fetch caches. + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + // Now serve a document signed by a different key. + _, otherPriv := makeKey(t) + fs.docBody = jsonObject(2, nil) + fs.sigBody = signDoc(t, otherPriv, fs.docBody) + fs.modified = true + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a bad signature returned no error") + } + if !res.FromCache { + t.Fatal("a bad signature did not fall back to cache") + } + if res.Version != 1 { + t.Fatalf("fallback Version = %d, want 1", res.Version) + } +} + +// A rollback — a version lower than the cached one — falls back to the cache. +func TestRollbackFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + // Cache version 5. + fs.setVersion(5) + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + // Serve version 3, still signed correctly. + fs.setVersion(3) + fs.modified = true + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a rollback returned no error") + } + if !res.FromCache { + t.Fatal("a rollback did not fall back to cache") + } + if res.Version != 5 { + t.Fatalf("fallback Version = %d, want 5", res.Version) + } +} + +// A dead source falls back to the cache. +func TestDeadSourceFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + // Kill the server. + srv.Close() + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a dead source returned no error") + } + if !res.FromCache { + t.Fatal("a dead source did not fall back to cache") + } +} + +// With no cached copy, a dead source returns a non-nil error and Doc nil. +func TestDeadSourceWithNoCacheReturnsErrorAndNilDoc(t *testing.T) { + pub, _ := makeKey(t) + p := &Puller{ + URL: "http://127.0.0.1:1/doc.json", // nothing listening + Keys: []ed25519.PublicKey{pub}, + CacheDir: t.TempDir(), + Budget: 200 * time.Millisecond, + } + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a dead source with no cache returned no error") + } + if res.Doc != nil { + t.Fatal("Doc should be nil with no cache") + } + if res.FromCache { + t.Fatal("FromCache should be false with no cache") + } +} + +// A document that is not JSON falls back to the cache. +func TestNonJSONDocumentFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + // Serve non-JSON but sign it so it gets past the signature check. + bad := []byte("not json at all") + fs.docBody = bad + fs.sigBody = signDoc(t, fs.priv, bad) + fs.modified = true + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("non-JSON returned no error") + } + if !res.FromCache { + t.Fatal("non-JSON did not fall back to cache") + } +} + +// A document that is not an object (an array) falls back to the cache. +func TestNonObjectDocumentFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + arr := []byte(`[1,2,3]`) + fs.docBody = arr + fs.sigBody = signDoc(t, fs.priv, arr) + fs.modified = true + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("non-object returned no error") + } + if !res.FromCache { + t.Fatal("non-object did not fall back to cache") + } +} + +// A document with no version falls back to the cache. +func TestDocumentWithNoVersionFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + obj := []byte(`{"notversion":1}`) + fs.docBody = obj + fs.sigBody = signDoc(t, fs.priv, obj) + fs.modified = true + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("no-version returned no error") + } + if !res.FromCache { + t.Fatal("no-version did not fall back to cache") + } +} + +// A document with a non-integer version falls back to the cache. +func TestDocumentWithNonIntegerVersionFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + obj := []byte(`{"version":"five"}`) + fs.docBody = obj + fs.sigBody = signDoc(t, fs.priv, obj) + fs.modified = true + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("non-integer version returned no error") + } + if !res.FromCache { + t.Fatal("non-integer version did not fall back to cache") + } +} + +// Extra top-level fields are none of this package's business. +func TestExtraFieldsArePassedThrough(t *testing.T) { + fs, srv := newFetchServer(t) + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: t.TempDir(), + } + extra := map[string]any{"min_client": "1.0", "submit": "https://example.com"} + fs.docBody = jsonObject(1, extra) + fs.sigBody = signDoc(t, fs.priv, fs.docBody) + fs.modified = true + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("extra fields caused an error: %v", err) + } + // The bytes handed back carry the extra fields. + var got map[string]any + if err := json.Unmarshal(res.Doc, &got); err != nil { + t.Fatal(err) + } + if _, ok := got["min_client"]; !ok { + t.Fatal("extra field min_client was stripped") + } +} + +// ----------------------------------------------------------------------------- + +// The cache lives in CacheDir and is written atomically. +func TestGoodFetchIsCachedOnDisk(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + doc, err := os.ReadFile(filepath.Join(cacheDir, "doc.json")) + if err != nil { + t.Fatalf("doc.json not cached: %v", err) + } + if !bytesEqual(doc, fs.docBody) { + t.Fatal("cached doc does not match source") + } + if _, err := os.ReadFile(filepath.Join(cacheDir, "doc.sig")); err != nil { + t.Fatalf("doc.sig not cached: %v", err) + } + if _, err := os.ReadFile(filepath.Join(cacheDir, "meta.json")); err != nil { + t.Fatalf("meta.json not cached: %v", err) + } +} + +// Cached files have mode 0600. +func TestCacheFilesHaveMode0600(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + for _, name := range []string{"doc.json", "doc.sig", "meta.json"} { + info, err := os.Stat(filepath.Join(cacheDir, name)) + if err != nil { + t.Fatalf("stat %s: %v", name, err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("%s has mode %o, want 0600", name, info.Mode().Perm()) + } + } +} + +// CacheDir is made if it is missing. +func TestCacheDirIsMadeIfMissing(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := filepath.Join(t.TempDir(), "nested", "cache") + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(cacheDir, "doc.json")); err != nil { + t.Fatalf("nested CacheDir was not made: %v", err) + } +} + +// A cache that does not read is no cache: corrupt bytes are treated as nothing +// cached, and the next good fetch replaces them. +func TestCorruptCacheIsTreatedAsEmpty(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + // Write garbage into the cache. + writeFile(t, filepath.Join(cacheDir, "doc.json"), []byte("garbage")) + writeFile(t, filepath.Join(cacheDir, "doc.sig"), []byte("garbage")) + writeFile(t, filepath.Join(cacheDir, "meta.json"), []byte("{}")) + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + } + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("a corrupt cache should not surface its own error: %v", err) + } + if res.FromCache { + t.Fatal("a corrupt cache was served as a cache hit") + } + if !res.Changed { + t.Fatal("replacing a corrupt cache should be Changed") + } +} + +// A cache whose signature no longer verifies under the current keys is no cache. +func TestCacheWithUntrustedSignatureIsTreatedAsEmpty(t *testing.T) { + _, oldPriv := makeKey(t) + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + // Cache a document signed by the old key. + doc := jsonObject(1, nil) + sig := signDoc(t, oldPriv, doc) + writeFile(t, filepath.Join(cacheDir, "doc.json"), doc) + writeFile(t, filepath.Join(cacheDir, "doc.sig"), sig) + writeFile(t, filepath.Join(cacheDir, "meta.json"), []byte(`{"version":1,"fetched_at":"2020-01-01T00:00:00Z"}`)) + // Pull with a different (untrusted) key set — the old signature must not + // verify, and the source is fetched fresh. + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + } + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("untrusted cache should fall through silently: %v", err) + } + if res.FromCache { + t.Fatal("an untrusted cache was served") + } +} + +// ----------------------------------------------------------------------------- + +// An empty URL is an error. +func TestEmptyURLError(t *testing.T) { + p := &Puller{CacheDir: t.TempDir()} + _, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("empty URL returned no error") + } +} + +// The whole call lives inside Budget. +func TestBudgetTimesOut(t *testing.T) { + fs, srv := newFetchServer(t) + fs.delay = 500 * time.Millisecond + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + Budget: 100 * time.Millisecond, + // An empty directory rather than no directory at all: there is nothing + // to fall back to either way, and a blank CacheDir reads doc.json + // RELATIVE TO THE WORKING DIRECTORY, so the test would answer from + // whatever happened to sit beside the source file. + CacheDir: t.TempDir(), + } + _, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a call over budget returned no error") + } +} + +// The call also lives inside ctx, whichever ends first. +func TestContextCancelEndsCall(t *testing.T) { + fs, srv := newFetchServer(t) + fs.delay = 500 * time.Millisecond + ctx, cancel := contextWithTimeout(50 * time.Millisecond) + defer cancel() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + Budget: 10 * time.Second, + CacheDir: t.TempDir(), + } + _, err := p.Pull(ctx) + if err == nil { + t.Fatal("a cancelled context returned no error") + } +} + +// Pull never panics, even on a wrong-length key. +func TestPullNeverPanicsOnBadKey(t *testing.T) { + pub, priv := makeKey(t) + dir := t.TempDir() + doc := jsonObject(1, nil) + sig := signDoc(t, priv, doc) + writeFile(t, filepath.Join(dir, "doc.json"), doc) + writeFile(t, filepath.Join(dir, "doc.json.sig"), sig) + bad := ed25519.PublicKey(make([]byte, 5)) + p := &Puller{ + URL: filepath.Join(dir, "doc.json"), + Keys: []ed25519.PublicKey{bad, pub}, + CacheDir: t.TempDir(), + } + // Must not panic; the bad key is skipped and the good one verifies. + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("Pull with a bad key failed: %v", err) + } + if res.Version != 1 { + t.Fatalf("Version = %d, want 1", res.Version) + } +} + +// ----------------------------------------------------------------------------- + +// A 304 with no cached copy is a failure: the source claimed not-modified for +// something we never had. +func Test304WithNoCacheIsAnError(t *testing.T) { + pub, priv := makeKey(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".sig") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(signDoc(t, priv, []byte{})) + return + } + w.WriteHeader(http.StatusNotModified) + })) + t.Cleanup(srv.Close) + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{pub}, + CacheDir: t.TempDir(), + } + _, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a 304 with no cache returned no error") + } +} + +// A non-200 status from the source falls back to the cache. +func TestNon200StatusFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + fs.statusDoc = http.StatusInternalServerError + fs.modified = true + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a 500 returned no error") + } + if !res.FromCache { + t.Fatal("a 500 did not fall back to cache") + } +} + +// A signature fetch that fails falls back to the cache. +func TestSignatureFetchFailureFallsBackToCache(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + fs.setVersion(2) + fs.modified = true + fs.statusSig = http.StatusNotFound + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a missing signature returned no error") + } + if !res.FromCache { + t.Fatal("a missing signature did not fall back to cache") + } +} + +// The cache stores the ETag and uses it on the next fetch. +func TestCacheStoresETag(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + meta, err := os.ReadFile(filepath.Join(cacheDir, "meta.json")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(meta), "v1") { + t.Fatalf("meta.json does not contain the etag: %s", meta) + } +} + +// No temp files are left behind after a good cache write. +func TestAtomicWriteLeavesNoTempFiles(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(cacheDir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".") { + t.Fatalf("temp file left behind: %s", e.Name()) + } + } +} + +// A cache whose meta.json is missing is treated as empty. +func TestCacheWithMissingMetaIsTreatedAsEmpty(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + // Write doc and sig but no meta. + doc := jsonObject(1, nil) + sig := signDoc(t, fs.priv, doc) + writeFile(t, filepath.Join(cacheDir, "doc.json"), doc) + writeFile(t, filepath.Join(cacheDir, "doc.sig"), sig) + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + } + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("missing meta surfaced an error: %v", err) + } + if res.FromCache { + t.Fatal("a cache missing meta.json was served from cache") + } +} + +// A file source whose document is missing returns an error with no cache. +func TestFileSourceMissingReturnsError(t *testing.T) { + pub, _ := makeKey(t) + p := &Puller{ + URL: filepath.Join(t.TempDir(), "nope.json"), + Keys: []ed25519.PublicKey{pub}, + CacheDir: t.TempDir(), + } + _, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a missing file source returned no error") + } +} + +// A file source falls back to cache when the file goes away. +func TestFileSourceFallsBackWhenFileDisappears(t *testing.T) { + pub, priv := makeKey(t) + dir := t.TempDir() + doc := jsonObject(3, nil) + sig := signDoc(t, priv, doc) + path := filepath.Join(dir, "doc.json") + writeFile(t, path, doc) + writeFile(t, path+".sig", sig) + cacheDir := t.TempDir() + p := &Puller{ + URL: path, + Keys: []ed25519.PublicKey{pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + os.Remove(path) + res, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("a missing file returned no error") + } + if !res.FromCache { + t.Fatal("a missing file did not fall back to cache") + } + if res.Version != 3 { + t.Fatalf("fallback Version = %d, want 3", res.Version) + } +} + +// A version equal to the cached one is good and Changed is false. +func TestEqualVersionIsGoodAndNotChanged(t *testing.T) { + fs, srv := newFetchServer(t) + cacheDir := t.TempDir() + p := &Puller{ + URL: srv.URL + "/doc.json", + Keys: []ed25519.PublicKey{fs.pub}, + CacheDir: cacheDir, + TTL: 0, + } + if _, err := p.Pull(t.Context()); err != nil { + t.Fatal(err) + } + // Serve the same version again with a fresh body (different object, same + // version number) so the 304 path is not taken. + fs.docBody = jsonObject(1, map[string]any{"note": "refreshed"}) + fs.sigBody = signDoc(t, fs.priv, fs.docBody) + fs.modified = true + res, err := p.Pull(t.Context()) + if err != nil { + t.Fatalf("equal version should be good: %v", err) + } + if res.Changed { + t.Fatal("equal version reported Changed true") + } +} + +// ----------------------------------------------------------------------------- + +// helpers + +func bytesEqual(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// contextWithTimeout makes a context with a timeout, because t.Context() does +// not take one. +func contextWithTimeout(d time.Duration) (context.Context, func()) { + return context.WithTimeout(context.Background(), d) +} + +// A cancelled context ends the call whatever the URL names. A file source is +// served in microseconds, so nothing here is about speed: it is about Pull +// meaning one thing on both roads. +func TestCancelEndsAFileSourceToo(t *testing.T) { + pub, priv := makeKey(t) + dir := t.TempDir() + doc := jsonObject(4, nil) + writeFile(t, filepath.Join(dir, "doc.json"), doc) + writeFile(t, filepath.Join(dir, "doc.json.sig"), signDoc(t, priv, doc)) + + ctx, cancel := contextWithTimeout(time.Hour) + cancel() + p := &Puller{ + URL: filepath.Join(dir, "doc.json"), + Keys: []ed25519.PublicKey{pub}, + CacheDir: t.TempDir(), + } + got, err := p.Pull(ctx) + if err == nil { + t.Fatal("a cancelled context read the file anyway") + } + if got.Doc != nil || got.Version != 0 || got.FromCache { + t.Fatalf("a cancelled call with no cache answered %+v", got) + } +} + +// A cache whose metadata does not describe the bytes beside it is no cache. +// The three files are committed by three renames, so this is what a run cut +// between them leaves: a document and a signature that verify as a pair, under +// the version of the copy before. +func TestACacheWhoseMetaDescribesOtherBytesIsNoCache(t *testing.T) { + pub, priv := makeKey(t) + cacheDir := t.TempDir() + doc := jsonObject(9, nil) + writeFile(t, filepath.Join(cacheDir, "doc.json"), doc) + writeFile(t, filepath.Join(cacheDir, "doc.sig"), signDoc(t, priv, doc)) + writeFile(t, filepath.Join(cacheDir, "meta.json"), + []byte(`{"etag":"","version":3,"fetched_at":"`+time.Now().UTC().Format(time.RFC3339Nano)+`"}`)) + + p := &Puller{ + URL: filepath.Join(t.TempDir(), "absent.json"), + Keys: []ed25519.PublicKey{pub}, + CacheDir: cacheDir, + TTL: time.Hour, + } + // A young cache would be answered from without a read; this one is not a + // cache at all, so the absent source is consulted and fails with nothing + // to fall back to. + got, err := p.Pull(t.Context()) + if err == nil { + t.Fatal("the torn cache was served") + } + if got.Doc != nil || got.Version != 0 || got.FromCache { + t.Fatalf("the torn cache answered %+v, want nothing at all", got) + } +} diff --git a/internal/pool/record/record.go b/internal/pool/record/record.go new file mode 100644 index 0000000000..551233202d --- /dev/null +++ b/internal/pool/record/record.go @@ -0,0 +1,215 @@ +// Package record turns a judge's seat scores into the two records an install +// keeps of them: additive cells in a tally sheet on disk, and one row per +// score in an outbox. +// +// THE MECHANISM. [Recorder.Record] takes the scores [judge.Judge] answered and +// does two things with them. Every valid one is observed into the sheet the +// recorder holds, under the role_quality metric the crew picker reads seat +// quality from, and one row is appended to the recorder's outbox when it holds +// one. A score whose role is not one of the judged seats, or that is not on +// the 0-100 scale, is skipped and named in the error; the others are recorded +// however the round went, and the sheet is observed whatever the outbox does. +// A nil outbox records locally only. +// +// The sheet an install keeps of its own scores is its own evidence: it is +// saved under the pool directory ([OwnSheetName]) and read back as the cells +// the crew picker's prior blends in beside an index's ([Cells]). +package record + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/Agent-Field/codeaf/internal/crewpick" + "github.com/Agent-Field/codeaf/internal/pool/judge" + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/pool/tally" +) + +// Metric is the sheet metric a judged seat score is observed under: the word +// the crew picker reads a measured seat quality from. +const Metric = "role_quality" + +// OwnSheetName is the file the install's own sheet is kept as, under the pool +// directory. +const OwnSheetName = "own.json" + +// OwnSheetPath is where the install's own sheet lives for the pool directory +// given. +func OwnSheetPath(poolDir string) string { + return filepath.Join(poolDir, OwnSheetName) +} + +// dirMode is the mode the pool's own directory keeps: private to the person. +const dirMode = 0700 + +// LoadSheet reads the sheet at path. A path with no file answers an empty +// sheet and no error — an install that has recorded nothing yet is not a +// fault; bytes that are there and do not parse as a sheet document are. +func LoadSheet(path string) (*tally.Sheet, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return tally.New(), nil + } + return nil, err + } + var s tally.Sheet + if err := json.Unmarshal(data, &s); err != nil { + return nil, fmt.Errorf("record: %s does not hold a sheet document: %w", path, err) + } + return &s, nil +} + +// SaveSheet writes the sheet to a temporary file in the path's own directory +// and renames it into place, so a reader mid-write never sees half a document. +// The file is mode 0600 and its parent directory is created with mode 0700 +// when it is missing. +func SaveSheet(path string, s *tally.Sheet) error { + doc, err := json.Marshal(s) + if err != nil { + return err + } + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, dirMode); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, OwnSheetName+".*") + if err != nil { + return err + } + name := tmp.Name() + defer os.Remove(name) // nothing once the rename has landed + if _, err := tmp.Write(append(doc, '\n')); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(name, path) +} + +// rowSchema is the only row schema this package writes. +const rowSchema = 1 + +// Row is one judged seat score as it leaves the install: what was scored — +// the metric, the seat and the model that held it, the score on the 0-100 +// scale — and the facts of the run it was read from, the judge that answered, +// the door the run came in by, the size of the crew, and the day. +type Row struct { + Schema int `json:"schema"` + Metric string `json:"metric"` + Role string `json:"role"` + Model string `json:"model"` + Score float64 `json:"score"` + Judge string `json:"judge"` + Door string `json:"door"` + Size string `json:"size"` + Day string `json:"day"` +} + +// RowsOf reads judge scores into rows, one per score. Every row carries +// schema 1 and the role_quality metric; the day is spelled by the caller, the +// way the outbox spells its own. +func RowsOf(scores []judge.Score, judgeModel, door, size, day string) []Row { + rows := make([]Row, 0, len(scores)) + for _, score := range scores { + rows = append(rows, Row{ + Schema: rowSchema, + Metric: Metric, + Role: string(score.Role), + Model: score.Model, + Score: score.Score, + Judge: judgeModel, + Door: door, + Size: size, + Day: day, + }) + } + return rows +} + +// judgedRoles is the whole set of seats a score may name: the three seats the +// judge reads, and nothing else. A role outside it is not a seat the picker +// reads a quality for. +var judgedRoles = map[judge.Role]bool{ + judge.RoleWorker: true, + judge.RoleHigh: true, + judge.RoleMastermind: true, +} + +// Recorder turns judge scores into the two records an install keeps: the +// sheet observed locally and, when one is held, the outbox rows that leave. +type Recorder struct { + // Sheet holds the install's own tallies; a recorder without one has + // nowhere to put what it is given, and refuses. + Sheet *tally.Sheet + // Outbox is the rows' way out. A nil Outbox keeps the scores locally + // only. + Outbox *outbox.Outbox +} + +// Record observes every score it is given into the sheet, and appends one row +// per score to the outbox when the recorder holds one. THE SHEET IS OBSERVED +// WHATEVER THE OUTBOX DOES: every score is observed before the first row is +// appended, so an append that fails costs the install nothing of its own +// evidence, and the first append error is returned once every score has been +// observed. A score whose role is not one of the judged seats, or whose score +// is not on the 0-100 scale, is skipped and named in the error beside the +// others that were recorded. +func (r *Recorder) Record(scores []judge.Score, judgeModel, door, size, day string) error { + if r.Sheet == nil { + return errors.New("record: no sheet to observe into") + } + rows := RowsOf(scores, judgeModel, door, size, day) + var errs []error + var appendErr error + for i, score := range scores { + if !judgedRoles[score.Role] { + errs = append(errs, fmt.Errorf("record: the %s seat is not a judged seat and was skipped", score.Role)) + continue + } + if score.Score < 0 || score.Score > 100 { + errs = append(errs, fmt.Errorf("record: the %s seat's score %v is outside 0-100 and was skipped", score.Role, score.Score)) + continue + } + r.Sheet.Observe(Metric, string(score.Role), score.Model, nil, score.Score) + if r.Outbox == nil { + continue + } + line, err := json.Marshal(rows[i]) + if err != nil { + continue // a struct of scalars never fails to marshal + } + if err := r.Outbox.Append(line); err != nil && appendErr == nil { + appendErr = err + } + } + if appendErr != nil { + errs = append(errs, appendErr) + } + return errors.Join(errs...) +} + +// Cells reads the sheet's own role_quality cells — the ones recorded with no +// dim labels — as the cells a prior reads: the seat, the model, the mean of +// the scores and the count behind them, sorted by seat then model, so the +// answer is a property of the sheet and never of the order it was observed +// in. A nil sheet answers no cells. +func Cells(s *tally.Sheet) []crewpick.Cell { + if s == nil { + return nil + } + var cells []crewpick.Cell + s.Each(Metric, func(role, model string, dims map[string]string, c tally.Cell) { + if dims != nil { + return + } + cells = append(cells, crewpick.Cell{Role: role, Model: model, Mean: c.Mean(), N: int(c.N)}) + }) + return cells +} diff --git a/internal/pool/record/record_test.go b/internal/pool/record/record_test.go new file mode 100644 index 0000000000..6508e8c266 --- /dev/null +++ b/internal/pool/record/record_test.go @@ -0,0 +1,328 @@ +package record + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/crewpick" + "github.com/Agent-Field/codeaf/internal/pool/judge" + "github.com/Agent-Field/codeaf/internal/pool/outbox" + "github.com/Agent-Field/codeaf/internal/pool/tally" +) + +// A path with no file behind it is an install that has recorded nothing yet: +// an empty sheet and no error, never a fault. +func TestALoadOfAMissingPathAnswersAnEmptySheet(t *testing.T) { + sheet, err := LoadSheet(filepath.Join(t.TempDir(), "absent", OwnSheetName)) + if err != nil { + t.Fatalf("a missing sheet loaded as %v", err) + } + if got := Cells(sheet); len(got) != 0 { + t.Fatalf("a missing sheet answered %d cells, want none", len(got)) + } +} + +// Bytes that are there and hold no sheet document are a fault, named. +func TestALoadOfUnparsableBytesIsAnError(t *testing.T) { + path := filepath.Join(t.TempDir(), OwnSheetName) + if err := os.WriteFile(path, []byte("not a document"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadSheet(path); err == nil { + t.Fatal("unparsable bytes loaded as a sheet") + } +} + +// A sheet saved is the sheet loaded: every cell comes back with the +// observations it held, however many addresses it kept. +func TestASaveThenLoadRoundTrips(t *testing.T) { + path := filepath.Join(t.TempDir(), "pool", OwnSheetName) + sheet := tally.New() + sheet.Observe(Metric, "worker", "a/one", nil, 80) + sheet.Observe(Metric, "worker", "a/one", nil, 90) + sheet.Observe(Metric, "high", "b/two", nil, 70) + sheet.Observe(Metric, "high", "b/two", map[string]string{"quant": "fp8"}, 55) + sheet.Win("worker", "a/one", "b/two") + if err := SaveSheet(path, sheet); err != nil { + t.Fatalf("save: %v", err) + } + back, err := LoadSheet(path) + if err != nil { + t.Fatalf("load: %v", err) + } + for _, want := range []struct { + role, model string + dims map[string]string + cell tally.Cell + }{ + {"worker", "a/one", nil, tally.Cell{N: 2, Sum: 170, SumSq: 14500}}, + {"high", "b/two", nil, tally.Cell{N: 1, Sum: 70, SumSq: 4900}}, + {"high", "b/two", map[string]string{"quant": "fp8"}, tally.Cell{N: 1, Sum: 55, SumSq: 3025}}, + } { + got, ok := back.Cell(Metric, want.role, want.model, want.dims) + if !ok || got != want.cell { + t.Fatalf("the %s/%s cell (%v) came back %+v ok %v, want %+v", want.role, want.model, want.dims, got, ok, want.cell) + } + } + if over, under := back.Wins("worker", "a/one", "b/two"); over != 1 || under != 0 { + t.Fatalf("the win came back %d over %d under, want 1 over 0", over, under) + } + + // The file is private, and the save is atomic enough to leave no temp + // file beside the sheet it wrote. + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("the sheet was saved with mode %v, want 0600", info.Mode().Perm()) + } + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != OwnSheetName { + t.Fatalf("the pool directory holds %d entries, want the sheet alone", len(entries)) + } +} + +// A second save over the first leaves the one file, whole: the rename +// replaces the sheet rather than appending to it. +func TestASaveOverAnExistingSheetReplacesIt(t *testing.T) { + path := filepath.Join(t.TempDir(), OwnSheetName) + first := tally.New() + first.Observe(Metric, "worker", "a/one", nil, 80) + if err := SaveSheet(path, first); err != nil { + t.Fatal(err) + } + second := tally.New() + second.Observe(Metric, "high", "b/two", nil, 70) + if err := SaveSheet(path, second); err != nil { + t.Fatal(err) + } + back, err := LoadSheet(path) + if err != nil { + t.Fatal(err) + } + if _, ok := back.Cell(Metric, "worker", "a/one", nil); ok { + t.Fatal("the replaced sheet still holds the first save's cell") + } + if _, ok := back.Cell(Metric, "high", "b/two", nil); !ok { + t.Fatal("the replaced sheet does not hold the second save's cell") + } +} + +// Every valid score is observed into the sheet and one row apiece is appended +// to the outbox, each row carrying the day, the judge and the door the run +// came in by. +func TestRecordObservesEveryScoreAndAppendsOneRowApiece(t *testing.T) { + sheet := tally.New() + box, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.jsonl")) + if err != nil { + t.Fatal(err) + } + defer box.Close() + recorder := &Recorder{Sheet: sheet, Outbox: box} + + scores := []judge.Score{ + {Role: judge.RoleWorker, Model: "a/one", Score: 80}, + {Role: judge.RoleHigh, Model: "b/two", Score: 66}, + {Role: judge.RoleMastermind, Model: "c/three", Score: 91}, + } + if err := recorder.Record(scores, "z/fourth", "chat", "three", "2026-09-17"); err != nil { + t.Fatalf("record: %v", err) + } + for _, want := range []struct { + role judge.Role + model string + mean float64 + }{ + {judge.RoleWorker, "a/one", 80}, + {judge.RoleHigh, "b/two", 66}, + {judge.RoleMastermind, "c/three", 91}, + } { + cell, ok := sheet.Cell(Metric, string(want.role), want.model, nil) + if !ok || cell.N != 1 || cell.Mean() != want.mean { + t.Fatalf("the %s seat observed %+v ok %v, want one observation of %v", want.role, cell, ok, want.mean) + } + } + pending := box.Pending() + if len(pending) != len(scores) { + t.Fatalf("%d rows appended, want one per score", len(pending)) + } + for i, row := range pending { + var got Row + if err := json.Unmarshal(row.Payload, &got); err != nil { + t.Fatalf("row %d is not a row: %v", i, err) + } + want := Row{ + Schema: 1, Metric: Metric, + Role: string(scores[i].Role), + Model: scores[i].Model, + Score: scores[i].Score, + Judge: "z/fourth", Door: "chat", Size: "three", Day: "2026-09-17", + } + if got != want { + t.Fatalf("row %d is %+v, want %+v", i, got, want) + } + } +} + +// RowsOf spells every row the same way: schema 1 under the role_quality +// metric, whatever the score. +func TestRowsOfSpellsEveryRowTheSameWay(t *testing.T) { + rows := RowsOf([]judge.Score{ + {Role: judge.RoleWorker, Model: "a/one", Score: 80, Reason: "it worked"}, + }, "z/fourth", "chat", "solo", "2026-09-17") + if len(rows) != 1 { + t.Fatalf("%d rows, want one", len(rows)) + } + if rows[0].Schema != 1 || rows[0].Metric != Metric || rows[0].Role != "worker" || + rows[0].Model != "a/one" || rows[0].Score != 80 || rows[0].Judge != "z/fourth" || + rows[0].Door != "chat" || rows[0].Size != "solo" || rows[0].Day != "2026-09-17" { + t.Fatalf("the row is %+v", rows[0]) + } + line, err := json.Marshal(rows[0]) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{`"schema":1`, `"metric":"role_quality"`, `"role":"worker"`, `"day":"2026-09-17"`} { + if !strings.Contains(string(line), key) { + t.Fatalf("the row's json is missing %s: %s", key, line) + } + } +} + +// A score whose seat is not one of the judged seats is skipped and named, and +// the scores beside it are recorded all the same. +func TestABadRoleIsSkippedAndNamedAndTheOthersLand(t *testing.T) { + for _, bad := range []judge.Score{ + {Role: "planner", Model: "a/one", Score: 80}, + {Role: "", Model: "a/one", Score: 80}, + {Role: judge.RoleWorker, Model: "a/one", Score: 140}, + {Role: judge.RoleWorker, Model: "a/one", Score: -1}, + } { + sheet := tally.New() + box, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.jsonl")) + if err != nil { + t.Fatal(err) + } + recorder := &Recorder{Sheet: sheet, Outbox: box} + err = recorder.Record([]judge.Score{ + bad, + {Role: judge.RoleWorker, Model: "a/one", Score: 80}, + }, "z/fourth", "chat", "two", "2026-09-17") + box.Close() + if err == nil { + t.Fatalf("a %s score at %v was recorded without a word", bad.Role, bad.Score) + } + if !strings.Contains(err.Error(), "skipped") { + t.Fatalf("the error does not say what was skipped: %v", err) + } + // The good score landed in both records; the bad one in neither. + if cells := Cells(sheet); len(cells) != 1 { + t.Fatalf("the sheet holds %d cells, want the good score's one: %+v", len(cells), cells) + } + if cell, ok := sheet.Cell(Metric, "worker", "a/one", nil); !ok || cell.N != 1 || cell.Mean() != 80 { + t.Fatalf("the good score did not land: %+v ok %v", cell, ok) + } + if pending := box.Pending(); len(pending) != 1 { + t.Fatalf("%d rows appended, want the good score's one", len(pending)) + } + } +} + +// An outbox that cannot take a row costs the install nothing of its own +// evidence: every score is observed into the sheet all the same, and the +// first append error is what comes back. +func TestRecordObservesTheSheetWhateverTheOutboxDoes(t *testing.T) { + sheet := tally.New() + box, err := outbox.Open(filepath.Join(t.TempDir(), "outbox.jsonl")) + if err != nil { + t.Fatal(err) + } + if err := box.Close(); err != nil { + t.Fatal(err) + } + recorder := &Recorder{Sheet: sheet, Outbox: box} + err = recorder.Record([]judge.Score{ + {Role: judge.RoleWorker, Model: "a/one", Score: 80}, + {Role: judge.RoleHigh, Model: "b/two", Score: 66}, + {Role: judge.RoleMastermind, Model: "c/three", Score: 91}, + }, "z/fourth", "chat", "three", "2026-09-17") + if err == nil { + t.Fatal("a closed outbox recorded without a word") + } + for _, want := range []struct { + role judge.Role + model string + mean float64 + }{ + {judge.RoleWorker, "a/one", 80}, + {judge.RoleHigh, "b/two", 66}, + {judge.RoleMastermind, "c/three", 91}, + } { + cell, ok := sheet.Cell(Metric, string(want.role), want.model, nil) + if !ok || cell.N != 1 || cell.Mean() != want.mean { + t.Fatalf("the %s seat was not observed despite the failed append: %+v ok %v", want.role, cell, ok) + } + } +} + +// A recorder with no outbox keeps the scores locally only, and says nothing. +func TestANilOutboxRecordsLocallyOnly(t *testing.T) { + sheet := tally.New() + recorder := &Recorder{Sheet: sheet} + err := recorder.Record([]judge.Score{ + {Role: judge.RoleWorker, Model: "a/one", Score: 80}, + {Role: judge.RoleHigh, Model: "b/two", Score: 66}, + }, "z/fourth", "chat", "two", "2026-09-17") + if err != nil { + t.Fatalf("a local-only record said %v", err) + } + if cell, ok := sheet.Cell(Metric, "worker", "a/one", nil); !ok || cell.N != 1 { + t.Fatalf("the worker score did not land: %+v ok %v", cell, ok) + } + if cell, ok := sheet.Cell(Metric, "high", "b/two", nil); !ok || cell.N != 1 { + t.Fatalf("the high score did not land: %+v ok %v", cell, ok) + } +} + +// A recorder with no sheet has nowhere to put what it is given, and says so. +func TestARecorderWithoutASheetRefuses(t *testing.T) { + recorder := &Recorder{} + if err := recorder.Record(nil, "z/fourth", "chat", "one", "2026-09-17"); err == nil { + t.Fatal("a sheetless recorder recorded") + } +} + +// The own sheet's cells answer sorted by seat then model, each carrying the +// mean of its observations and the count behind it, and a cell recorded under +// dim labels is not one of them — the prior reads the sheet's plain cells. +func TestCellsAnswerSortedAndCarryTheMeanAndCount(t *testing.T) { + sheet := tally.New() + // Observed deliberately out of the order they must answer in. + sheet.Observe(Metric, "worker", "b/later", nil, 60) + sheet.Observe(Metric, "worker", "a/first", nil, 90) + sheet.Observe(Metric, "worker", "a/first", nil, 80) + sheet.Observe(Metric, "high", "c/top", nil, 70) + sheet.Observe(Metric, "high", "a/first", map[string]string{"quant": "fp8"}, 40) + + got := Cells(sheet) + if len(got) != 3 { + t.Fatalf("%d cells answered, want the three plain ones: %+v", len(got), got) + } + want := []crewpick.Cell{ + {Role: "high", Model: "c/top", Mean: 70, N: 1}, + {Role: "worker", Model: "a/first", Mean: 85, N: 2}, + {Role: "worker", Model: "b/later", Mean: 60, N: 1}, + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("cell %d is %+v, want %+v", i, got[i], want[i]) + } + } +} diff --git a/internal/pool/tally/tally.go b/internal/pool/tally/tally.go new file mode 100644 index 0000000000..1688e6d886 --- /dev/null +++ b/internal/pool/tally/tally.go @@ -0,0 +1,438 @@ +// Package tally keeps additive sufficient statistics for observations so +// that two sheets recorded on different machines can be merged by addition, +// in any order, and give the same answer as one sheet that saw everything. +// It is standard library only and touches no network, no disk and no clock. +// The only strings a sheet holds are the metric, role, model and dim labels +// an observation carries. +package tally + +import ( + "encoding/json" + "fmt" + "maps" + "math" + "slices" + "strconv" + "strings" + "sync" +) + +// maxNameLen bounds every label a sheet stores. +const maxNameLen = 128 + +// validName reports whether s may be stored: at most maxNameLen bytes, no +// newline, and no path separator — except that a model id may carry the +// single "/" between vendor and name. +func validName(s string, model bool) bool { + if len(s) > maxNameLen { + return false + } + slashes := 0 + for i := 0; i < len(s); i++ { + switch s[i] { + case '\n', '\\': + return false + case '/': + if !model { + return false + } + slashes++ + if slashes > 1 { + return false + } + } + } + return true +} + +// lp prefixes s with its byte length, so an address key built by +// concatenating labels is unambiguous whatever bytes the labels hold. +func lp(s string) string { + return strconv.Itoa(len(s)) + ":" + s +} + +// dimsKey canonicalises the dim labels of an address: keys sorted, every +// key and value length-prefixed. Nil and an empty map give the same key, +// and the order the keys were inserted in never matters. +func dimsKey(dims map[string]string) string { + keys := slices.Sorted(maps.Keys(dims)) + var b strings.Builder + for _, k := range keys { + b.WriteString(lp(k)) + b.WriteString(lp(dims[k])) + } + return b.String() +} + +// cellAddr is the full address of a cell, kept so the sheet can name it +// again in a document. +type cellAddr struct { + metric, role, model string + dims map[string]string +} + +// key is the unambiguous form of the address. +func (a cellAddr) key() string { + return lp(a.metric) + lp(a.role) + lp(a.model) + dimsKey(a.dims) +} + +type cellEntry struct { + addr cellAddr + cell Cell +} + +// winKey addresses one direction of a paired comparison. +type winKey struct { + role, winner, loser string +} + +// Sheet keeps additive sufficient statistics: one cell per address of +// metric, role, model and dims, and paired-comparison tallies per role. +// A Sheet is safe for concurrent use by multiple goroutines, and the zero +// value is an empty sheet ready to use. +type Sheet struct { + mu sync.Mutex + cells map[string]cellEntry + wins map[winKey]int64 +} + +// New returns an empty sheet. +func New() *Sheet { + return &Sheet{} +} + +// copyDims returns a private copy of dims, nil for nil or empty. +func copyDims(dims map[string]string) map[string]string { + if len(dims) == 0 { + return nil + } + out := make(map[string]string, len(dims)) + for k, v := range dims { + out[k] = v + } + return out +} + +// Observe records one observation x of metric for role and model, under +// the optional dim labels (for example quant=fp8). Nil dims and an empty +// map are the same address, and the order the labels were inserted in +// never matters. An observation that is NaN or infinite is ignored, and so +// is the whole call when any label is longer than 128 bytes or contains a +// newline or a path separator other than the single "/" a model id carries +// between vendor and name. +func (s *Sheet) Observe(metric, role, model string, dims map[string]string, x float64) { + if math.IsNaN(x) || math.IsInf(x, 0) { + return + } + if !validName(metric, false) || !validName(role, false) || !validName(model, true) { + return + } + for k, v := range dims { + if !validName(k, false) || !validName(v, false) { + return + } + } + a := cellAddr{metric: metric, role: role, model: model, dims: copyDims(dims)} + k := a.key() + s.mu.Lock() + defer s.mu.Unlock() + if s.cells == nil { + s.cells = make(map[string]cellEntry) + } + e, ok := s.cells[k] + if !ok { + e.addr = a + } + e.cell.N++ + e.cell.Sum += x + e.cell.SumSq += x * x + s.cells[k] = e +} + +// Cell returns the cell recorded for the address, and whether any +// observation was recorded for it. Nil dims and an empty map are the same +// address, and the order the labels were inserted in never matters. +func (s *Sheet) Cell(metric, role, model string, dims map[string]string) (Cell, bool) { + k := cellAddr{metric: metric, role: role, model: model, dims: dims}.key() + s.mu.Lock() + defer s.mu.Unlock() + e, ok := s.cells[k] + return e.cell, ok +} + +// Each hands every cell recorded under metric to fn, one call per cell, in a +// deterministic order: role, then model, then the dim labels. Cells recorded +// under other metrics are not handed over, and the sheet is only read — fn +// sees the dim labels as the sheet stores them, nil when the observations +// carried none, and it does not run while the sheet is locked, so it may read +// the sheet again. +func (s *Sheet) Each(metric string, fn func(role, model string, dims map[string]string, c Cell)) { + s.mu.Lock() + seen := make([]cellEntry, 0, len(s.cells)) + for _, e := range s.cells { + if e.addr.metric == metric { + seen = append(seen, e) + } + } + s.mu.Unlock() + slices.SortFunc(seen, func(a, b cellEntry) int { + if c := strings.Compare(a.addr.role, b.addr.role); c != 0 { + return c + } + if c := strings.Compare(a.addr.model, b.addr.model); c != 0 { + return c + } + return strings.Compare(dimsKey(a.addr.dims), dimsKey(b.addr.dims)) + }) + for _, e := range seen { + fn(e.addr.role, e.addr.model, e.addr.dims, e.cell) + } +} + +// Win records one paired comparison in which winner beat loser for role. +// The call is ignored when a name is empty, when winner and loser are +// equal, or when a name is longer than 128 bytes or contains a newline or +// a path separator other than the single "/" a model id carries between +// vendor and name. +func (s *Sheet) Win(role, winner, loser string) { + if role == "" || winner == "" || loser == "" || winner == loser { + return + } + if !validName(role, false) || !validName(winner, true) || !validName(loser, true) { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.wins == nil { + s.wins = make(map[winKey]int64) + } + s.wins[winKey{role: role, winner: winner, loser: loser}]++ +} + +// Wins returns the paired comparisons recorded for role in both directions +// between a and b. +func (s *Sheet) Wins(role, a, b string) (aOverB, bOverA int64) { + s.mu.Lock() + defer s.mu.Unlock() + return s.wins[winKey{role: role, winner: a, loser: b}], + s.wins[winKey{role: role, winner: b, loser: a}] +} + +// winCount is one snapshot entry of the paired-comparison tallies. +type winCount struct { + key winKey + n int64 +} + +// Merge adds other into s: afterwards s holds what it held plus what +// other holds, and other is left unmodified. Merging a sheet into itself +// doubles it. A nil sheet on either side is a no-op. +func (s *Sheet) Merge(other *Sheet) { + if s == nil || other == nil { + return + } + if other == s { + s.mu.Lock() + cells, wins := s.snapshotLocked() + s.applyLocked(cells, wins) + s.mu.Unlock() + return + } + other.mu.Lock() + cells, wins := other.snapshotLocked() + other.mu.Unlock() + + s.mu.Lock() + s.applyLocked(cells, wins) + s.mu.Unlock() +} + +// snapshotLocked copies the sheet's cells and wins. +func (s *Sheet) snapshotLocked() ([]cellEntry, []winCount) { + cells := make([]cellEntry, 0, len(s.cells)) + for _, e := range s.cells { + cells = append(cells, e) + } + wins := make([]winCount, 0, len(s.wins)) + for k, n := range s.wins { + wins = append(wins, winCount{key: k, n: n}) + } + return cells, wins +} + +// applyLocked adds snapshot cells and wins to the sheet. +func (s *Sheet) applyLocked(cells []cellEntry, wins []winCount) { + if s.cells == nil { + s.cells = make(map[string]cellEntry, len(cells)) + } + for _, e := range cells { + k := e.addr.key() + cur, ok := s.cells[k] + if !ok { + cur = cellEntry{addr: cellAddr{ + metric: e.addr.metric, + role: e.addr.role, + model: e.addr.model, + dims: copyDims(e.addr.dims), + }} + } + cur.cell.N += e.cell.N + cur.cell.Sum += e.cell.Sum + cur.cell.SumSq += e.cell.SumSq + s.cells[k] = cur + } + if s.wins == nil { + s.wins = make(map[winKey]int64, len(wins)) + } + for _, w := range wins { + s.wins[w.key] += w.n + } +} + +// Cell is the sufficient statistic recorded for one address: how many +// observations were seen, their sum, and the sum of their squares. Two +// cells for the same address add field by field. +type Cell struct { + N int64 + Sum float64 + SumSq float64 +} + +// Mean returns the mean of the observations, or 0 when the cell is empty. +func (c Cell) Mean() float64 { + if c.N == 0 { + return 0 + } + return c.Sum / float64(c.N) +} + +// Var returns the sample variance of the observations. It is 0 when the +// cell holds fewer than two observations, and it is never negative: +// rounding in the sum-of-squares formula is clamped to zero. +func (c Cell) Var() float64 { + if c.N < 2 { + return 0 + } + v := (c.SumSq - c.Sum*c.Sum/float64(c.N)) / float64(c.N-1) + if v < 0 { + return 0 + } + return v +} + +// cellDoc and winDoc are the JSON shapes of one cell and one win; doc is +// the whole document. Fields unknown to this package, here and inside an +// entry, are ignored when reading. +type cellDoc struct { + Metric string `json:"metric"` + Role string `json:"role"` + Model string `json:"model"` + Dims map[string]string `json:"dims"` + N int64 `json:"n"` + Sum float64 `json:"sum"` + SumSq float64 `json:"sumsq"` +} + +type winDoc struct { + Role string `json:"role"` + Winner string `json:"winner"` + Loser string `json:"loser"` + Wins int64 `json:"wins"` +} + +type doc struct { + Schema int `json:"schema"` + Cells []cellDoc `json:"cells"` + Wins []winDoc `json:"wins"` +} + +// schemaVersion is the only document schema this package writes and reads. +const schemaVersion = 1 + +// MarshalJSON encodes the sheet deterministically: sheets holding the same +// statistics give byte-identical documents, whatever order anything was +// inserted in. Cells are sorted by metric, role, model and dim labels, +// wins by role, winner and loser, dim labels by key. The document carries +// a top-level "schema": 1. +func (s *Sheet) MarshalJSON() ([]byte, error) { + s.mu.Lock() + cells, wins := s.snapshotLocked() + s.mu.Unlock() + + docs := make([]cellDoc, 0, len(cells)) + for _, e := range cells { + dims := e.addr.dims + if dims == nil { + dims = map[string]string{} + } + docs = append(docs, cellDoc{ + Metric: e.addr.metric, + Role: e.addr.role, + Model: e.addr.model, + Dims: dims, + N: e.cell.N, + Sum: e.cell.Sum, + SumSq: e.cell.SumSq, + }) + } + slices.SortFunc(docs, func(a, b cellDoc) int { + if c := strings.Compare(a.Metric, b.Metric); c != 0 { + return c + } + if c := strings.Compare(a.Role, b.Role); c != 0 { + return c + } + if c := strings.Compare(a.Model, b.Model); c != 0 { + return c + } + return strings.Compare(dimsKey(a.Dims), dimsKey(b.Dims)) + }) + wdocs := make([]winDoc, 0, len(wins)) + for _, w := range wins { + wdocs = append(wdocs, winDoc{Role: w.key.role, Winner: w.key.winner, Loser: w.key.loser, Wins: w.n}) + } + slices.SortFunc(wdocs, func(a, b winDoc) int { + if c := strings.Compare(a.Role, b.Role); c != 0 { + return c + } + if c := strings.Compare(a.Winner, b.Winner); c != 0 { + return c + } + return strings.Compare(a.Loser, b.Loser) + }) + return json.Marshal(doc{Schema: schemaVersion, Cells: docs, Wins: wdocs}) +} + +// UnmarshalJSON replaces the sheet's contents with the document. Fields +// unknown to this package, at the top level and inside an entry, are +// ignored. A document whose schema is greater than 1 is refused with an +// error. +func (s *Sheet) UnmarshalJSON(data []byte) error { + var d doc + if err := json.Unmarshal(data, &d); err != nil { + return err + } + if d.Schema > schemaVersion { + return fmt.Errorf("tally: document schema %d is newer than this build reads", d.Schema) + } + cells := make(map[string]cellEntry, len(d.Cells)) + for _, c := range d.Cells { + a := cellAddr{metric: c.Metric, role: c.Role, model: c.Model, dims: copyDims(c.Dims)} + k := a.key() + e := cells[k] + e.addr = a + e.cell.N += c.N + e.cell.Sum += c.Sum + e.cell.SumSq += c.SumSq + cells[k] = e + } + wins := make(map[winKey]int64, len(d.Wins)) + for _, w := range d.Wins { + wins[winKey{role: w.Role, winner: w.Winner, loser: w.Loser}] += w.Wins + } + s.mu.Lock() + s.cells = cells + s.wins = wins + s.mu.Unlock() + return nil +} diff --git a/internal/pool/tally/tally_test.go b/internal/pool/tally/tally_test.go new file mode 100644 index 0000000000..ca772e3212 --- /dev/null +++ b/internal/pool/tally/tally_test.go @@ -0,0 +1,571 @@ +package tally + +import ( + "bytes" + "encoding/json" + "math" + "strconv" + "strings" + "sync" + "testing" +) + +func TestCellMeanAndVar(t *testing.T) { + if got := (Cell{}).Mean(); got != 0 { + t.Fatalf("empty Mean = %v, want 0", got) + } + if got := (Cell{}).Var(); got != 0 { + t.Fatalf("empty Var = %v, want 0", got) + } + if got := (Cell{N: 1, Sum: 5, SumSq: 25}).Var(); got != 0 { + t.Fatalf("single-observation Var = %v, want 0", got) + } + c := Cell{N: 3, Sum: 6, SumSq: 14} // observations 1, 2, 3 + if got := c.Mean(); got != 2 { + t.Fatalf("Mean = %v, want 2", got) + } + if got := c.Var(); got != 1 { + t.Fatalf("Var = %v, want 1", got) + } +} + +func TestCellVarNeverNegative(t *testing.T) { + // (4 - 3*3/2) / 1 rounds negative; the clamp must hold it at zero. + if got := (Cell{N: 2, Sum: 3, SumSq: 4}).Var(); got != 0 { + t.Fatalf("Var = %v, want 0", got) + } +} + +func checkCell(t *testing.T, s *Sheet, metric, role, model string, dims map[string]string, want Cell) { + t.Helper() + got, ok := s.Cell(metric, role, model, dims) + if !ok { + t.Fatalf("Cell(%q, %q, %q, %v) not recorded", metric, role, model, dims) + } + if got != want { + t.Fatalf("Cell(%q, %q, %q, %v) = %+v, want %+v", metric, role, model, dims, got, want) + } +} + +func checkNoCell(t *testing.T, s *Sheet, metric, role, model string, dims map[string]string) { + t.Helper() + if got, ok := s.Cell(metric, role, model, dims); ok { + t.Fatalf("Cell(%q, %q, %q, %v) = %+v, want nothing recorded", metric, role, model, dims, got) + } +} + +func TestObserveAccumulates(t *testing.T) { + s := New() + s.Observe("tok", "coder", "vendor/model", nil, 2) + s.Observe("tok", "coder", "vendor/model", nil, 4) + checkCell(t, s, "tok", "coder", "vendor/model", nil, Cell{N: 2, Sum: 6, SumSq: 20}) + c, _ := s.Cell("tok", "coder", "vendor/model", nil) + if c.Mean() != 3 || c.Var() != 2 { + t.Fatalf("Mean = %v, Var = %v, want 3 and 2", c.Mean(), c.Var()) + } + s.Observe("tok", "coder", "vendor/model", nil, -1.5) + checkCell(t, s, "tok", "coder", "vendor/model", nil, Cell{N: 3, Sum: 4.5, SumSq: 22.25}) +} + +func TestDimsAddressing(t *testing.T) { + s := New() + s.Observe("m", "r", "model", nil, 1) + s.Observe("m", "r", "model", map[string]string{}, 1) + s.Observe("m", "r", "model", map[string]string{"quant": "fp8"}, 1) + s.Observe("m", "r", "model", map[string]string{"quant": "fp8"}, 1) // same address again + s.Observe("m", "r", "model", map[string]string{"a": "1", "b": "2"}, 1) + s.Observe("m", "r", "model", map[string]string{"b": "2", "a": "1"}, 1) // order never matters + checkCell(t, s, "m", "r", "model", nil, Cell{N: 2, Sum: 2, SumSq: 2}) + checkCell(t, s, "m", "r", "model", map[string]string{"quant": "fp8"}, Cell{N: 2, Sum: 2, SumSq: 2}) + checkCell(t, s, "m", "r", "model", map[string]string{"b": "2", "a": "1"}, Cell{N: 2, Sum: 2, SumSq: 2}) + checkNoCell(t, s, "m", "r", "model", map[string]string{"quant": "fp4"}) + checkNoCell(t, s, "m", "r", "model", map[string]string{"quant": "fp8", "extra": "x"}) + s.Observe("m", "r2", "model", nil, 1) + s.Observe("m2", "r", "model", nil, 1) + s.Observe("m", "r", "model2", nil, 1) + checkCell(t, s, "m", "r2", "model", nil, Cell{N: 1, Sum: 1, SumSq: 1}) + checkCell(t, s, "m2", "r", "model", nil, Cell{N: 1, Sum: 1, SumSq: 1}) + checkCell(t, s, "m", "r", "model2", nil, Cell{N: 1, Sum: 1, SumSq: 1}) +} + +func TestAddressingIsUnambiguous(t *testing.T) { + // Labels that would collide in a plain concatenation stay distinct. + s := New() + s.Observe("a:b", "c", "m", nil, 1) + s.Observe("a", "b:c", "m", nil, 1) + checkCell(t, s, "a:b", "c", "m", nil, Cell{N: 1, Sum: 1, SumSq: 1}) + checkCell(t, s, "a", "b:c", "m", nil, Cell{N: 1, Sum: 1, SumSq: 1}) +} + +func TestObserveIgnoresNonFinite(t *testing.T) { + s := New() + s.Observe("m", "r", "model", nil, math.NaN()) + s.Observe("m", "r", "model", nil, math.Inf(1)) + s.Observe("m", "r", "model", nil, math.Inf(-1)) + checkNoCell(t, s, "m", "r", "model", nil) + s.Observe("m", "r", "model", nil, 1) // the sheet still records after refusals + checkCell(t, s, "m", "r", "model", nil, Cell{N: 1, Sum: 1, SumSq: 1}) +} + +func TestObserveRefusesBadNames(t *testing.T) { + long := strings.Repeat("x", maxNameLen+1) + cases := []struct { + name string + metric, role, mod string + dims map[string]string + }{ + {"metric with newline", "to\nk", "r", "m", nil}, + {"long metric", long, "r", "m", nil}, + {"metric with path separator", "to/k", "r", "m", nil}, + {"role with path separator", "m", "ro/le", "m", nil}, + {"long role", "m", long, "m", nil}, + {"long model", "m", "r", long, nil}, + {"model with backslash", "m", "r", `ven\dor`, nil}, + {"model with two separators", "m", "r", "ven/dor/name", nil}, + {"dim key with separator", "m", "r", "m", map[string]string{"qu/ant": "fp8"}}, + {"long dim key", "m", "r", "m", map[string]string{long: "fp8"}}, + {"long dim value", "m", "r", "m", map[string]string{"quant": long}}, + {"dim value with newline", "m", "r", "m", map[string]string{"quant": "fp\n8"}}, + } + for _, tc := range cases { + s := New() + s.Observe(tc.metric, tc.role, tc.mod, tc.dims, 1) + s.mu.Lock() + got := len(s.cells) + s.mu.Unlock() + if got != 0 { + t.Fatalf("%s: recorded a cell", tc.name) + } + } + // At the bound a label is fine, and the single separator a model id + // carries between vendor and name is fine. + s := New() + s.Observe(strings.Repeat("x", maxNameLen), "r", "vendor/name", nil, 1) + s.Observe("plain", "r", "bare", nil, 1) + checkCell(t, s, strings.Repeat("x", maxNameLen), "r", "vendor/name", nil, Cell{N: 1, Sum: 1, SumSq: 1}) + checkCell(t, s, "plain", "r", "bare", nil, Cell{N: 1, Sum: 1, SumSq: 1}) +} + +func TestWinAndWins(t *testing.T) { + s := New() + s.Win("coder", "a", "b") + s.Win("coder", "a", "b") + s.Win("coder", "b", "a") + aOverB, bOverA := s.Wins("coder", "a", "b") + if aOverB != 2 || bOverA != 1 { + t.Fatalf("Wins = %d, %d, want 2, 1", aOverB, bOverA) + } + aOverB, bOverA = s.Wins("coder", "b", "a") // the query is symmetric + if aOverB != 1 || bOverA != 2 { + t.Fatalf("Wins = %d, %d, want 1, 2", aOverB, bOverA) + } + if x, y := s.Wins("reviewer", "a", "b"); x != 0 || y != 0 { + t.Fatalf("wins leak across roles: %d, %d", x, y) + } + if x, y := s.Wins("coder", "a", "c"); x != 0 || y != 0 { + t.Fatalf("unrecorded pair: %d, %d", x, y) + } + // A model id may carry its separator here too. + s.Win("coder", "vendor/a", "vendor/b") + if x, y := s.Wins("coder", "vendor/a", "vendor/b"); x != 1 || y != 0 { + t.Fatalf("Wins = %d, %d, want 1, 0", x, y) + } +} + +func TestWinRefusesBadCalls(t *testing.T) { + s := New() + s.Win("", "a", "b") + s.Win("r", "", "b") + s.Win("r", "a", "") + s.Win("r", "a", "a") // winner == loser + s.Win("r", "a\n", "b") + s.Win("r", "a", "b\n") + s.Win("r", "a/b/c", "b") // more than the one separator a model id carries + s.Win("r", "a", `b\c`) // a backslash is a path separator + s.Win("ro/le", "a", "b") // a role carries none + s.Win("r", "a", strings.Repeat("b", maxNameLen+1)) + if x, y := s.Wins("r", "a", "b"); x != 0 || y != 0 { + t.Fatalf("refused Win recorded: %d, %d", x, y) + } +} + +func sameJSON(t *testing.T, a, b *Sheet) bool { + t.Helper() + ja, err := json.Marshal(a) + if err != nil { + t.Fatalf("marshal: %v", err) + } + jb, err := json.Marshal(b) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return bytes.Equal(ja, jb) +} + +func TestMergeCommutative(t *testing.T) { + a := New() + a.Observe("m", "r", "model", map[string]string{"quant": "fp8"}, 3) + a.Win("r", "x", "y") + b := New() + b.Observe("m", "r", "model", nil, 4) + b.Observe("m", "r", "model", nil, 5) + b.Win("r", "y", "x") + + ab := New() + ab.Merge(a) + ab.Merge(b) + ba := New() + ba.Merge(b) + ba.Merge(a) + if !sameJSON(t, ab, ba) { + t.Fatal("merge is not commutative") + } +} + +func TestMergeAssociative(t *testing.T) { + a := New() + a.Observe("m", "r", "model", nil, 1) + a.Win("r", "x", "y") + b := New() + b.Observe("m", "r", "model", map[string]string{"q": "1"}, 2) + c := New() + c.Observe("m", "r", "model", nil, 3) + c.Observe("m", "r", "model", nil, 4) + c.Win("r", "y", "x") + + ab := New() + ab.Merge(a) + ab.Merge(b) + abC := New() + abC.Merge(ab) + abC.Merge(c) + + bc := New() + bc.Merge(b) + bc.Merge(c) + aBC := New() + aBC.Merge(a) + aBC.Merge(bc) + + if !sameJSON(t, abC, aBC) { + t.Fatal("merge is not associative") + } +} + +func TestMergeSelfDoubles(t *testing.T) { + s := New() + s.Observe("m", "r", "model", nil, 2) + s.Observe("m", "r", "model", map[string]string{"q": "1"}, 5) + s.Win("r", "a", "b") + s.Win("r", "b", "a") + s.Merge(s) + checkCell(t, s, "m", "r", "model", nil, Cell{N: 2, Sum: 4, SumSq: 8}) + checkCell(t, s, "m", "r", "model", map[string]string{"q": "1"}, Cell{N: 2, Sum: 10, SumSq: 50}) + if x, y := s.Wins("r", "a", "b"); x != 2 || y != 2 { + t.Fatalf("Wins = %d, %d, want 2, 2", x, y) + } +} + +func TestMergeNilIsNoOp(t *testing.T) { + s := New() + s.Observe("m", "r", "model", nil, 1) + before, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + s.Merge(nil) + if j, _ := json.Marshal(s); !bytes.Equal(j, before) { + t.Fatal("Merge(nil) changed the sheet") + } + var nilSheet *Sheet + nilSheet.Merge(s) // must not panic + nilSheet.Merge(nil) // must not panic either +} + +func TestMergeLeavesOtherAlone(t *testing.T) { + other := New() + other.Observe("m", "r", "model", nil, 7) + other.Observe("m", "r", "model", map[string]string{"q": "1"}, 1) + other.Win("r", "p", "q") + before, err := json.Marshal(other) + if err != nil { + t.Fatal(err) + } + s := New() + s.Merge(other) + if j, _ := json.Marshal(other); !bytes.Equal(j, before) { + t.Fatal("Merge modified the merged-in sheet") + } + checkCell(t, s, "m", "r", "model", nil, Cell{N: 1, Sum: 7, SumSq: 49}) + checkCell(t, s, "m", "r", "model", map[string]string{"q": "1"}, Cell{N: 1, Sum: 1, SumSq: 1}) + if x, y := s.Wins("r", "p", "q"); x != 1 || y != 0 { + t.Fatalf("Wins = %d, %d, want 1, 0", x, y) + } +} + +func TestMergedSheetsEqualSingleSheet(t *testing.T) { + one := New() + one.Observe("tok", "coder", "vendor/m", nil, 1) + one.Observe("tok", "coder", "vendor/m", nil, 2) + one.Observe("tok", "coder", "vendor/m", map[string]string{"quant": "fp8"}, 7) + one.Observe("latency", "reviewer", "vendor/m", nil, 0.5) + one.Win("coder", "a", "b") + one.Win("coder", "b", "a") + one.Win("reviewer", "a", "b") + + // The same observations and wins, split over three sheets and recorded + // in a different order. + parts := []*Sheet{New(), New(), New()} + parts[2].Observe("latency", "reviewer", "vendor/m", nil, 0.5) + parts[0].Observe("tok", "coder", "vendor/m", nil, 2) + parts[1].Observe("tok", "coder", "vendor/m", map[string]string{"quant": "fp8"}, 7) + parts[0].Win("coder", "b", "a") + parts[1].Win("reviewer", "a", "b") + parts[0].Observe("tok", "coder", "vendor/m", nil, 1) + parts[1].Win("coder", "a", "b") + + merged := New() + merged.Merge(parts[2]) + merged.Merge(parts[0]) + merged.Merge(parts[1]) + if !sameJSON(t, merged, one) { + t.Fatal("merged sheets disagree with one sheet that saw everything") + } +} + +func TestMarshalEmpty(t *testing.T) { + j, err := json.Marshal(New()) + if err != nil { + t.Fatal(err) + } + if want := `{"schema":1,"cells":[],"wins":[]}`; string(j) != want { + t.Fatalf("empty sheet = %s, want %s", j, want) + } +} + +func TestMarshalDeterministic(t *testing.T) { + s1 := New() + s1.Observe("b", "r", "m", nil, 1) + s1.Observe("a", "r", "m", nil, 1) + s1.Observe("m", "r", "m1", map[string]string{"b": "2"}, 1) + s1.Observe("m", "r", "m1", map[string]string{"a": "1"}, 2) + s1.Win("r", "y", "x") + s1.Win("r", "x", "y") + + s2 := New() + s2.Observe("a", "r", "m", nil, 1) + s2.Observe("b", "r", "m", nil, 1) + s2.Observe("m", "r", "m1", map[string]string{"a": "1"}, 2) + s2.Observe("m", "r", "m1", map[string]string{"b": "2"}, 1) + s2.Win("r", "x", "y") + s2.Win("r", "y", "x") + + j1, err := json.Marshal(s1) + if err != nil { + t.Fatal(err) + } + j2, err := json.Marshal(s2) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(j1, j2) { + t.Fatalf("marshal depends on insertion order:\n%s\n%s", j1, j2) + } + if i, k := strings.Index(string(j1), `"metric":"a"`), strings.Index(string(j1), `"metric":"b"`); i > k { + t.Fatalf("cells not sorted: %s", j1) + } + if i, k := strings.Index(string(j1), `"winner":"x"`), strings.Index(string(j1), `"winner":"y"`); i > k { + t.Fatalf("wins not sorted: %s", j1) + } +} + +func TestRoundTrip(t *testing.T) { + s := New() + s.Observe("tok", "coder", "vendor/m", map[string]string{"quant": "fp8"}, 1.5) + s.Observe("tok", "coder", "vendor/m", nil, 2) + s.Win("coder", "a", "b") + j, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + back := New() + if err := json.Unmarshal(j, back); err != nil { + t.Fatal(err) + } + if !sameJSON(t, back, s) { + t.Fatalf("round trip changed the sheet: %s", j) + } + checkCell(t, back, "tok", "coder", "vendor/m", map[string]string{"quant": "fp8"}, Cell{N: 1, Sum: 1.5, SumSq: 2.25}) + checkCell(t, back, "tok", "coder", "vendor/m", nil, Cell{N: 1, Sum: 2, SumSq: 4}) + if x, y := back.Wins("coder", "a", "b"); x != 1 || y != 0 { + t.Fatalf("Wins = %d, %d, want 1, 0", x, y) + } +} + +func TestUnmarshalReplacesContents(t *testing.T) { + src := New() + src.Observe("m", "r", "model", nil, 1) + j, err := json.Marshal(src) + if err != nil { + t.Fatal(err) + } + dirty := New() + dirty.Observe("old", "r", "model", nil, 9) + dirty.Win("r", "x", "y") + if err := json.Unmarshal(j, dirty); err != nil { + t.Fatal(err) + } + if !sameJSON(t, dirty, src) { + t.Fatal("UnmarshalJSON did not replace existing contents") + } + if _, ok := dirty.Cell("old", "r", "model", nil); ok { + t.Fatal("old contents survived UnmarshalJSON") + } +} + +func TestUnmarshalIgnoresUnknownFields(t *testing.T) { + doc := `{"schema":1,"note":"hello","cells":[` + + `{"metric":"m","role":"r","model":"vendor/m","dims":{"quant":"fp8"},"n":2,"sum":3,"sumsq":5,"taken_by":"x"},` + + `{"metric":"m2","role":"r","model":"vendor/m","n":1,"sum":2,"sumsq":4}` + + `],"wins":[{"role":"r","winner":"a","loser":"b","wins":3,"when":12}],"extra":[1]}` + s := New() + if err := json.Unmarshal([]byte(doc), s); err != nil { + t.Fatal(err) + } + checkCell(t, s, "m", "r", "vendor/m", map[string]string{"quant": "fp8"}, Cell{N: 2, Sum: 3, SumSq: 5}) + checkCell(t, s, "m2", "r", "vendor/m", nil, Cell{N: 1, Sum: 2, SumSq: 4}) + if x, y := s.Wins("r", "a", "b"); x != 3 || y != 0 { + t.Fatalf("Wins = %d, %d, want 3, 0", x, y) + } +} + +func TestUnmarshalRefusesNewerSchema(t *testing.T) { + if err := json.Unmarshal([]byte(`{"schema":2}`), New()); err == nil { + t.Fatal("schema 2 accepted") + } + if err := json.Unmarshal([]byte(`{"schema":99,"cells":[]}`), New()); err == nil { + t.Fatal("schema 99 accepted") + } + if err := json.Unmarshal([]byte(`{"schema":1,"cells":[],"wins":[]}`), New()); err != nil { + t.Fatalf("schema 1 refused: %v", err) + } + if err := json.Unmarshal([]byte(`{}`), New()); err != nil { + t.Fatalf("document without a schema refused: %v", err) + } +} + +func TestUnmarshalInvalidJSON(t *testing.T) { + for _, bad := range []string{``, `{`, `{"schema":1,"cells":"no"}`, `{"schema":"1"}`} { + if err := json.Unmarshal([]byte(bad), New()); err == nil { + t.Fatalf("invalid document %q accepted", bad) + } + } +} + +func TestConcurrentUse(t *testing.T) { + const goroutines = 8 + const each = 100 + s := New() + + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < each; i++ { + s.Observe("m", "r", "model", nil, 1) + s.Observe("m", "r", "model", map[string]string{"g": strconv.Itoa(g)}, 2) + s.Win("r", "a", "b") + } + }(g) + } + // A writer merging while the others observe, and a reader walking the + // sheet the whole time. + part := New() + part.Observe("m", "r", "model", map[string]string{"part": "1"}, 3) + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 50; i++ { + s.Merge(part) + } + }() + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 200; i++ { + s.Wins("r", "a", "b") + s.Cell("m", "r", "model", nil) + if j, err := json.Marshal(s); err != nil { + t.Error(err) + return + } else if len(j) == 0 { + t.Error("empty marshal") + return + } + } + }() + wg.Wait() + + checkCell(t, s, "m", "r", "model", nil, Cell{N: goroutines * each, Sum: goroutines * each, SumSq: goroutines * each}) + for g := 0; g < goroutines; g++ { + checkCell(t, s, "m", "r", "model", map[string]string{"g": strconv.Itoa(g)}, + Cell{N: each, Sum: 2 * each, SumSq: 4 * each}) + } + checkCell(t, s, "m", "r", "model", map[string]string{"part": "1"}, Cell{N: 50, Sum: 150, SumSq: 450}) + if x, y := s.Wins("r", "a", "b"); x != goroutines*each || y != 0 { + t.Fatalf("Wins = %d, %d, want %d, 0", x, y, goroutines*each) + } +} + +// Each hands a metric's cells over one call apiece, in a settled order, and +// nothing under another metric and nothing the call does can move the sheet. +func TestEachHandsAMetricsCellsOverInASettledOrder(t *testing.T) { + s := New() + // Inserted deliberately out of order: the order Each answers must not + // carry the order the cells were observed in. + s.Observe("m", "worker", "b/late", nil, 2) + s.Observe("m", "high", "a/early", map[string]string{"quant": "fp8"}, 9) + s.Observe("m", "worker", "a/first", nil, 1) + s.Observe("other", "worker", "a/first", nil, 5) + s.Observe("m", "worker", "a/first", map[string]string{"quant": "fp8"}, 3) + + var seen []string + s.Each("m", func(role, model string, dims map[string]string, c Cell) { + word := role + " " + model + if dims == nil { + word += " nil" + } else { + word += " " + dims["quant"] + } + seen = append(seen, word) + switch { + case role == "high": + if c.N != 1 || c.Sum != 9 { + t.Errorf("the high cell is %+v, want one observation of 9", c) + } + case model == "a/first" && dims == nil: + if c.N != 1 || c.Sum != 1 { + t.Errorf("the first cell is %+v, want one observation of 1", c) + } + case model == "b/late": + if c.N != 1 || c.Sum != 2 { + t.Errorf("the late cell is %+v, want one observation of 2", c) + } + } + }) + want := []string{"high a/early fp8", "worker a/first nil", "worker a/first fp8", "worker b/late nil"} + if len(seen) != len(want) { + t.Fatalf("Each handed over %d cells (%v), want %d", len(seen), seen, len(want)) + } + for i := range want { + if seen[i] != want[i] { + t.Fatalf("the %d cell handed over is %q, want %q — role, then model, then the dim labels", i, seen[i], want[i]) + } + } + + // Nil dims and an empty map are the same address, so a second observation + // under an empty map lands in the cell Each already handed over. + s.Observe("m", "worker", "a/first", map[string]string{}, 4) + checkCell(t, s, "m", "worker", "a/first", nil, Cell{N: 2, Sum: 5, SumSq: 17}) +} diff --git a/internal/provider/client.go b/internal/provider/client.go index 9739e7e289..477d03bb99 100644 --- a/internal/provider/client.go +++ b/internal/provider/client.go @@ -2512,6 +2512,19 @@ func (e *APIError) FromUpstream() bool { return e != nil && strings.TrimSpace(e.Provider) != "" } +// UpstreamFault reports that THE NAMED ENDPOINT FAILED ON ITS OWN ACCOUNT: it +// was asked, it answered, and its answer was a fault of its own (5xx). It is the +// one relayed refusal that names a lane worth steering the SAME request away +// from on its next attempt: a relayed 4xx is that endpoint's reading of the +// request, which the next endpoint may read the same way, and a 429 is pacing +// with its own patience (retry.go). +// +// It lives here because a status is a fact only this package and the taxonomy +// may read; a caller asks the question and never the number. +func (e *APIError) UpstreamFault() bool { + return e.FromUpstream() && e.Status >= 500 +} + // OurRequest reports that THE REQUEST IS WHAT IS WRONG, so no endpoint will do // better with it. // diff --git a/internal/provider/refusal_test.go b/internal/provider/refusal_test.go index d54f71f2e3..a76586b712 100644 --- a/internal/provider/refusal_test.go +++ b/internal/provider/refusal_test.go @@ -53,6 +53,31 @@ func TestAProviderRefusalCarriesTheUpstreamAndItsOwnWords(t *testing.T) { } } +// A LANE TO AVOID IS A NAMED ENDPOINT THAT FAILED ON ITS OWN ACCOUNT. The +// retry asks this question rather than reading the status itself. +func TestOnlyANamedUpstreamsOwnFaultNamesALaneToAvoid(t *testing.T) { + fault := &APIError{Status: 502, Message: "upstream broke", Provider: "Alpha"} + if !fault.UpstreamFault() { + t.Error("a relayed 502 named no lane to avoid") + } + refused := &APIError{Status: 400, Message: "too long", Provider: "Alpha"} + if refused.UpstreamFault() { + t.Error("a relayed 400 was read as the endpoint's own fault") + } + paced := &APIError{Status: 429, Message: "slow down", Provider: "Alpha"} + if paced.UpstreamFault() { + t.Error("relayed pacing was read as the endpoint's own fault") + } + router := &APIError{Status: 502, Message: "bad gateway"} + if router.UpstreamFault() { + t.Error("a 502 that named nobody was blamed on an upstream") + } + var none *APIError + if none.UpstreamFault() { + t.Error("a nil refusal named a lane") + } +} + // AND THE TWO KINDS OF 4xx ARE TOLD APART BY SHAPE, NEVER BY A STATUS LIST. // // An upstream refused: another endpoint may serve, so the ladder goes on. The diff --git a/internal/resident/exhaustedresult_test.go b/internal/resident/exhaustedresult_test.go index 83b7e4fcbe..6f58e1f8c5 100644 --- a/internal/resident/exhaustedresult_test.go +++ b/internal/resident/exhaustedresult_test.go @@ -139,3 +139,65 @@ func TestAnExhaustedResultWithNothingRecordedFails(t *testing.T) { t.Fatalf("the failure %q does not name the exhaustion", final.Error) } } + +// AND A RE-DISPATCH THAT BANKED NOTHING ITS PREDECESSOR HAD NOT IS REFUSED, so +// the node fails rather than buying the same truncated ending a third time. The +// grant moves with the attempt now (see cmd/codeaf's regrantAfterRunningOut), +// which makes the comparison honest: a re-dispatch given more room that +// reaches the same recorded count has been paid to move and did not. +func TestAnExhaustedRedispatchThatBankedNothingNewFails(t *testing.T) { + graph := openRunnerStore(t) + spliceOneLeaf(t, graph, "task-2") + recordTurns(t, graph, "task-2", 3) + + starts := make(chan struct{}, 8) + runner := NewRunner(graph, func(ctx context.Context, node store.Node) (ExecResult, error) { + starts <- struct{}{} + // The same truncated ending both times: nothing new is recorded between + // the two attempts, so the re-dispatch reaches exactly what the attempt + // before it had already banked. + return ExecResult{ + Summary: "All 722 tests pass. Let me verify the dry-run tests specifically:", + Stop: executor.StopBudget, + Meter: executor.Meter{Name: "cost", Reached: 178086, Allowed: 176834, Unit: "tokens of billed work"}, + }, nil + }, "chat-runner", 1) + + ctx, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + served := make(chan error, 1) + go func() { served <- runner.Serve(ctx) }() + + waitFor(t, starts, "the leaf never started") + waitFor(t, starts, "the re-dispatch never started — the exhausted node was not sent round again") + deadline := time.Now().Add(8 * time.Second) + var final store.Node + for time.Now().Before(deadline) { + node, found, err := graph.Node("task-2") + if err != nil { + t.Fatalf("read the node: %v", err) + } + if found && node.Status == store.Failed { + final = node + break + } + time.Sleep(10 * time.Millisecond) + } + stop() + <-served + runner.Wait() + + if final.Status != store.Failed { + t.Fatalf("a re-dispatch that banked nothing new settled %q, want failed — it must not "+ + "be sent round to buy the same truncated ending again", final.Status) + } + if final.Attempt != 2 { + t.Fatalf("the node was claimed %d times, want the two of a first attempt and one refused re-dispatch", + final.Attempt) + } + for _, want := range []string{"refused", "added nothing"} { + if !contains(final.Error, want) { + t.Fatalf("the failure %q does not say the re-dispatch was refused for adding nothing", final.Error) + } + } +} diff --git a/internal/resident/grow.go b/internal/resident/grow.go index 16274281b8..a3e7f4c2bb 100644 --- a/internal/resident/grow.go +++ b/internal/resident/grow.go @@ -113,6 +113,14 @@ const ( // is not a cap and not a count: it is this job's OWN measured pace read // against the clock it is actually running under. CauseOutOfWall = "out-of-wall" + + // CauseSpendShare is the round refused because growth has already spent more + // than the requested work itself cost, measured from the moment a gate first + // found that work done. It is a bound against the bill rather than against a + // count of rounds, and it exists because a count could not stop a run whose + // named fix landed at minute eleven from spending the rest of its wall and + // most of its bill on growth. + CauseSpendShare = "spend-share" ) // The refusals in the words a person reads. They are constants because two of @@ -142,8 +150,19 @@ const ( // it was going to end with: a judgement on what it did. So the job stops // growing while there is still time to finish and be judged. RefusedOutOfWall = "there is not enough time left on this run to finish another round of work, so this is handed over while there is still time to check it" + + // The refusal that bounds growth against the bill. Once the requested work + // is done, a round that would take the total past twice what that work cost + // is refused and the job is handed over: the work the person asked for is + // finished, and what is left is growth they did not ask for. + RefusedSpendShare = "the requested work is done and growth has already doubled what it cost — handing over what's done" ) +// growthSpendShare bounds how much a job may spend on growth after its +// requested work is first found done: one means growth may at most double the +// bill that got the work done. +const growthSpendShare = 1.0 + // GrowthStopped reports whether a refusal cause is one of the two the governor // reaches by READING THE WORLD rather than by counting — the round before this // one changed nothing and neither did the work before that, or the remaining @@ -652,6 +671,20 @@ func growJob(ctx context.Context, graph *store.Store, ask Satisfier, req GrowReq } } + // 3b. Share of spend. Once the requested work is first found done, growth is + // bounded against the bill: a round is refused when more than + // [growthSpendShare] times the spend that got the work done has gone on + // growth since. The moment is the delivery gate's own first "done" — a + // pass or a coverage finding — and not this round's, so the rail holds + // across every later round. A job whose gate has never found its core + // work done, and a job with no session to bill, have no anchor and the + // rail does not apply: it refuses nothing it cannot measure. + if reach, err := growthShareReached(graph, req); err != nil { + return GrowVerdict{Round: round}, err + } else if reach { + return refuse(CauseSpendShare, RefusedSpendShare) + } + // 4. The only question that can say "there is nothing left to do". // // IT DOES NOT GET TO REFUSE THE FIRST ROUND AFTER A REVIEW FOUND @@ -715,6 +748,45 @@ func growJob(ctx context.Context, graph *store.Store, ask Satisfier, req GrowReq return GrowVerdict{Allow: true, Round: round, CoveredDespite: covered, Spent: req.spent}, nil } +// growthShareReached answers whether a job has already spent more on growth +// than [growthSpendShare] times what the requested work cost, measured from the +// moment a gate first found that work done. The read is the one the daily rail +// uses — SpendSinceSeq over the errand's session — so the two rails bill the +// same rows and a run cannot read cheap here and dear there. A job with no gate +// that found its work done, or with no session to bill, has no anchor and the +// rail does not apply; a read that fails is returned rather than guessed at. +func growthShareReached(graph *store.Store, req GrowRequest) (bool, error) { + if graph == nil { + return false, nil + } + jobRoot := strings.TrimSpace(req.JobRoot) + if jobRoot == "" { + jobRoot = req.Node.ID + } + session := strings.TrimSpace(req.Node.Provenance.SessionID) + if session == "" { + return false, nil + } + anchor, _, ok, err := graph.DeliveryGateAnchor(jobRoot) + if err != nil || !ok { + return false, err + } + // The share is read against the spend UP TO the anchor: the total billed so + // far minus what has been billed since the gate. One query per window, both + // off the rail's own read, so the two halves cannot come from two different + // readings of the same journal. + total, err := graph.SpendSinceSeq(session, 0) + if err != nil { + return false, err + } + after, err := graph.SpendSinceSeq(session, anchor) + if err != nil { + return false, err + } + before := total.Cost() - after.Cost() + return after.Cost() > growthSpendShare*before, nil +} + // admitGrowth journals a round that actually landed. It is called after the // splice rather than at the verdict, because the round counter is derived from // these events and an admission that never spliced is a round nobody spent — diff --git a/internal/resident/growshare_test.go b/internal/resident/growshare_test.go new file mode 100644 index 0000000000..ec0764724b --- /dev/null +++ b/internal/resident/growshare_test.go @@ -0,0 +1,67 @@ +package resident + +import ( + "context" + "testing" + + "github.com/Agent-Field/codeaf/internal/store" +) + +// shareRequest is the ordinary overrun round every cap is tested against: one +// node's worth of work, measured, that the free checks would otherwise admit. +func shareRequest(t *testing.T, graph *store.Store) GrowRequest { + t.Helper() + node := jobNode(t, graph, "job") + node.Provenance.SessionID = "s1" + return GrowRequest{JobRoot: "job", Node: node, Reason: GrowOverrun, Adding: 1, Measured: true, Produced: 1} +} + +// GROWTH IS REFUSED ONCE IT HAS SPENT MORE THAN THE REQUESTED WORK COST. The +// rail is measured from the moment a gate first found the work done, and it is +// a share of the bill rather than a count of rounds — the bound a count could +// not give a run whose named fix landed early and whose growth came after. +func TestGrowthIsRefusedWhenSpendAfterTheGateDoublesTheBill(t *testing.T) { + graph := crowdedJob(t, "s1", 3) + // The spend that got the requested work done, and the gate that found it + // done. + if err := graph.RecordUsage(store.NodeUsage{NodeID: "job", Cost: 1.0}); err != nil { + t.Fatal(err) + } + if err := graph.RecordDeliveryGate("job", store.DeliveryGate{Pass: true}); err != nil { + t.Fatal(err) + } + // And the growth since, which has already spent twice the work cost. + if err := graph.RecordUsage(store.NodeUsage{NodeID: "job", Cost: 2.0}); err != nil { + t.Fatal(err) + } + + verdict, err := growJob(context.Background(), graph, nil, shareRequest(t, graph)) + if err != nil { + t.Fatal(err) + } + if verdict.Allow || verdict.Cause != CauseSpendShare { + t.Fatalf("growth past the share was admitted: %+v", verdict) + } + if verdict.Refused != RefusedSpendShare { + t.Fatalf("the refusal is not the share rail's own words: %q", verdict.Refused) + } +} + +// THE RAIL ONLY APPLIES WHERE IT CAN BE MEASURED. A job whose gate has never +// found its core work done has no anchor, so growth is bounded by the caps and +// the rail refuses nothing — the fail-safe direction, because a rail that +// guessed at an anchor would stop work it cannot account for. +func TestAJobWithNoGateIsNotRefusedByTheShareRail(t *testing.T) { + graph := crowdedJob(t, "s2", 3) + if err := graph.RecordUsage(store.NodeUsage{NodeID: "job", Cost: 5.0}); err != nil { + t.Fatal(err) + } + + verdict, err := growJob(context.Background(), graph, nil, shareRequest(t, graph)) + if err != nil { + t.Fatal(err) + } + if !verdict.Allow || verdict.Cause == CauseSpendShare { + t.Fatalf("a job with no gate was refused by the share rail: %+v", verdict) + } +} diff --git a/internal/resident/overrun.go b/internal/resident/overrun.go index fd4fbb6475..ab5763f73d 100644 --- a/internal/resident/overrun.go +++ b/internal/resident/overrun.go @@ -543,6 +543,8 @@ func governorPhrase(cause string) string { return "no more room" case CauseCovered: return "already covered" + case CauseSpendShare: + return "the bill has doubled" } return "handing over" } diff --git a/internal/resident/ranoutwords.go b/internal/resident/ranoutwords.go index e451dfe18f..2876861b8a 100644 --- a/internal/resident/ranoutwords.go +++ b/internal/resident/ranoutwords.go @@ -6,8 +6,8 @@ import ( executor "github.com/Agent-Field/codeaf/internal/exec" ) -// The two sentences a node gets when the leaf holding it ran out. They are in -// one file because they are one account of one event told to two different +// The three sentences a node gets when the leaf holding it ran out. They are in +// one file because they are one account of one event told to three different // endings, and a reader who meets them a day apart must not have to work out // whether they are describing the same thing. // @@ -57,6 +57,18 @@ func outOfRoomFailure(result ExecResult, recorded int) string { return ranOutWords(result.Stop) + tail + meterAside(result.Meter) } +// outOfRoomRefusedFailure is the same fact when the re-dispatch that just ran +// out banked nothing the attempt before it had not banked: more room was the +// only thing a re-dispatch buys, the work did not move with it, and the node is +// failed here rather than paid to reach the same point again. It says the +// refusal and why, because a bare ending would read as a verdict on the work +// rather than on buying a second copy of the same truncation. +func outOfRoomRefusedFailure(result ExecResult, recorded int) string { + return ranOutWords(result.Stop) + " again, " + pluralTurns(recorded) + + " in and no further than the attempt before it, so the re-dispatch was refused: it added nothing" + + meterAside(result.Meter) +} + // A RELEASE REASON THAT HANDS WORK ON ENDS WITH THE SAME CLAUSE, AND IT IS // SPELLED HERE ONCE. // diff --git a/internal/resident/runner.go b/internal/resident/runner.go index 69db0201fd..349678771f 100644 --- a/internal/resident/runner.go +++ b/internal/resident/runner.go @@ -1420,6 +1420,16 @@ func (r *Runner) runOne(ctx context.Context, node store.Node, hold *leafHold) { // with nothing able to continue it has said all it is going to say, and it // is failed with the ending named rather than handed round again for ever. // + // AND A RE-DISPATCH THAT BANKED NOTHING NEW IS REFUSED. More room is the + // only thing a re-dispatch buys — the grant moves with the attempt now (see + // cmd/codeaf's regrantAfterRunningOut) — so a re-dispatch that reaches no + // further than the attempt before it has been paid to move and did not, and + // it is failed with the refusal named rather than sent round to buy the + // same truncated ending again. The count it is measured against is the one + // the hand-on release journaled: that row is the record's own answer to + // what the attempt before banked, and reading it beats keeping a second + // copy on the claim. + // // A PASSED DELIVERY GATE SETTLES THE NODE. Running out is a statement about // RESOURCES — the meter reached its bound while the worker was still going — // and a gate pass is a statement about the WORK, taken by reading the result @@ -1444,6 +1454,15 @@ func (r *Runner) runOne(ctx context.Context, node store.Node, hold *leafHold) { return } _, recorded := BankedRun(r.graph, node.ID) + // A read of the hand-on release that fails answers "nobody could say", + // which is the answer that leaves the claim to the rules below. + if node.Attempt > 0 { + if carried, handed, readErr := r.graph.ReleasedTurnsFor(node.ID); readErr == nil && + handed && recorded > 0 && recorded <= carried { + _ = r.graph.Fail(claim, outOfRoomRefusedFailure(result, recorded)) + return + } + } if node.Attempt < MaxOverrunRounds && recorded > 0 { _ = r.graph.ReleaseWithRecord(claim, outOfRoomClaimReason(result, recorded), recorded) return diff --git a/internal/revision/judge.go b/internal/revision/judge.go index 2eaa6343d9..7064504e96 100644 --- a/internal/revision/judge.go +++ b/internal/revision/judge.go @@ -2127,6 +2127,12 @@ func GapHandover(gaps string, revised bool, refused string) string { return handover } +// maxUnexercisedRounds bounds how many rounds one unexercised-only finding may +// buy. One is enough: the round exists to add checks for the behaviours the +// request states, and a second round aimed at the same finding measures nothing +// the first did not. +const maxUnexercisedRounds = 1 + // ExtendForGap is the authority the delivery gate never had. // // The judgement at the job root was already the right one and its maximum power @@ -2231,6 +2237,22 @@ func ExtendForGap(ctx context.Context, graph *store.Store, node store.Node, part if settled, met := metExtension(ctx, extension, unmet); met { return settled } + // ONE COVERAGE ROUND, AND NO MORE. An unexercised-only finding names + // behaviours the request states that no check exercises or asserts, and the + // round it buys exists to write those checks. That it STANDS until a + // measurement closes it is right — a behaviour an earlier round proved + // nothing exercises does not become exercised because this round took no + // reading — but it is also what let the same finding raise the same round at + // every later gate with nothing to bound it: a run whose named fix was + // committed minutes in spent the rest of its wall and most of its bill + // writing more tests, because a coverage round could be bought again and + // again. One is enough. See maxUnexercisedRounds. + if unexercisedOnly(unmet) && coverageRoundAlreadyBought(graph, base) { + extension.Refused = "the coverage round was already bought once, and the behaviours " + + "with no check that exercises them are left to the person" + extension.Unclosed = true + return extension + } if refusal := outOfWall(ctx, node); refusal != "" { extension.Refused, extension.Unclosed = refusal, true return extension @@ -2280,6 +2302,58 @@ func ExtendForGap(ctx context.Context, graph *store.Store, node store.Node, part return extension } +// unexercisedOnly says this finding is the coverage measurement and nothing +// else: behaviours the request states that no check exercises or asserts, with +// no other gap a round could be aimed at. +// +// A MODEL JUDGE'S OWN PROSE IS NOT ONE OF THESE. A judgement that still carries +// one is unsourced — the citation it names has not been weighed — so it is not +// unexercised-only however many behaviours ride on it; the round it buys is for +// the prose, and the coverage finding travels along. measuredHalf is what turns +// such a judgement into an unexercised-only one, and it is the same door every +// other measured finding goes through. +func unexercisedOnly(unmet Judgment) bool { + if len(unmet.Unexercised) == 0 && len(unmet.Unasserted) == 0 { + return false + } + if !unmet.Sourced { + return false + } + return !unmet.Mechanical && len(unmet.Constraint) == 0 && len(unmet.Consumers) == 0 && + len(unmet.Unbound) == 0 && len(unmet.OwnFailing) == 0 +} + +// coverageRoundAlreadyBought answers whether one of this job's earlier gates +// already grew work for an unexercised-only finding. The gate row carries the +// finding in Unexercised and Unasserted, and Extended says a round actually +// landed for it — the round counter is derived from these rows, so the ledger +// that bounds the next round is the one every other reader already replays. +// A read that fails answers no, which is the fail-safe direction: it buys the +// round rather than refusing one on evidence it could not see. +func coverageRoundAlreadyBought(graph *store.Store, base string) bool { + if graph == nil { + return false + } + gates, err := graph.DeliveryGateLineage(base) + if err != nil { + log.Printf("note: could not read the coverage ledger for %s: %v", base, err) + return false + } + bought := 0 + for _, gate := range gates { + if !gate.Extended || gate.Mechanical { + continue + } + if len(gate.Consumers) > 0 || len(gate.Unbound) > 0 || len(gate.OwnFailing) > 0 || len(gate.Constraint) > 0 { + continue + } + if len(gate.Unexercised) > 0 || len(gate.Unasserted) > 0 { + bought++ + } + } + return bought >= maxUnexercisedRounds +} + // coverageRefused reads the growth journal for the refusal that has just // happened and answers whether it was the coverage question, so the delivery // can say which governor declined in that governor's own terms. diff --git a/internal/revision/unexercisedbound_test.go b/internal/revision/unexercisedbound_test.go new file mode 100644 index 0000000000..42aad6b3ec --- /dev/null +++ b/internal/revision/unexercisedbound_test.go @@ -0,0 +1,97 @@ +package revision + +// One coverage round, and no more. + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/store" +) + +// coverageRoundJob splices one job into a private store so a gate can be read +// back off its ledger. +func coverageRoundJob(t *testing.T, session string) *store.Store { + t.Helper() + graph, err := store.Open(filepath.Join(t.TempDir(), "coverage-round.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { graph.Close() }) + if err := graph.Splice(store.RootID, store.Subtree{Nodes: []store.NodeSpec{{ + ID: "task-1", Brief: "add the flag", + }}}, store.Provenance{Origin: store.OriginUser, Intent: "add the flag", SessionID: session}); err != nil { + t.Fatal(err) + } + return graph +} + +// anUnexercisedFinding is the finding the acceptance gate raises on its own: +// behaviours the request states that no check exercises, sourced because nobody +// has to ask for them to be checked. +func anUnexercisedFinding() Judgment { + return Judgment{Pass: false, Checked: true, Sourced: true, + Gaps: "1 the request states has no check that exercises it.", + Unexercised: []string{"the flag turns on"}} +} + +// A SECOND UNEXERCISED-ONLY FINDING BUYS NO ROUND. The finding stands until a +// measurement closes it, so without this bound it raises the same round at every +// later gate — the run whose named fix was committed minutes in and spent the +// rest of its wall writing more tests. One round is bought for it and no more, +// and the finding is recorded unclosed so the delivery says why. +func TestASecondUnexercisedFindingBuysNoRound(t *testing.T) { + graph := coverageRoundJob(t, "s1") + // The earlier round: the request's behaviours had no check, a round was + // bought for it, and the row says so. + if err := graph.RecordDeliveryGate("task-1", store.DeliveryGate{ + Gap: "2 the request states have no check that exercises them.", + Unexercised: []string{"the flag turns on", "the flag turns off"}, + Round: 1, Extended: true, + }); err != nil { + t.Fatal(err) + } + + node := store.Node{ID: "task-1", Provenance: store.Provenance{Intent: "add the flag"}} + planned := 0 + extension := ExtendForGap(context.Background(), graph, node, "the flag is added", + anUnexercisedFinding(), nil, 10, + func(context.Context, string, string) (store.Subtree, error) { + planned++ + return store.Subtree{}, nil + }) + + if planned != 0 { + t.Fatalf("a second coverage round was planned: %d calls", planned) + } + if extension.Spliced != 0 || !extension.Unclosed { + t.Fatalf("the finding did not stay open: %+v", extension) + } + if !strings.Contains(extension.Refused, "coverage round was already bought once") { + t.Fatalf("the refusal does not say what it refused: %q", extension.Refused) + } +} + +// THE BOUND IS THE SECOND ROUND. A job that has never bought a coverage round +// buys its first, which is the whole point of the round existing; the bound may +// not refuse work the finding has not already paid for. +func TestAFirstUnexercisedFindingStillBuysItsRound(t *testing.T) { + graph := coverageRoundJob(t, "s2") + node := store.Node{ID: "task-1", Provenance: store.Provenance{Intent: "add the flag"}} + planned := 0 + extension := ExtendForGap(context.Background(), graph, node, "the flag is added", + anUnexercisedFinding(), nil, 10, + func(context.Context, string, string) (store.Subtree, error) { + planned++ + return store.Subtree{}, nil + }) + + if planned != 1 { + t.Fatalf("the first coverage round was not even planned: %d calls", planned) + } + if strings.Contains(extension.Refused, "coverage round was already bought once") { + t.Fatalf("the bound refused a round the finding had not paid for: %q", extension.Refused) + } +} diff --git a/internal/session/checkpoint.go b/internal/session/checkpoint.go index 88061efc3a..9416c079dd 100644 --- a/internal/session/checkpoint.go +++ b/internal/session/checkpoint.go @@ -1934,7 +1934,7 @@ func (a *Agent) readMark(ctx context.Context) checkpointRead { read.ownerGone = owner.Err() != nil return read } - a.addAuxiliaryUsage(response, reader, 1) + a.addAuxiliaryUsageAs(response, reader, 1, string(roles.RoleMarkReader)) if response.Usage != nil { read.costUSD = costOf(response.Usage) } @@ -3810,7 +3810,7 @@ func (a *Agent) readRemains(ctx context.Context) readerLine { if err != nil || response == nil { return readerLine{unreachable: true} } - a.addAuxiliaryUsage(response, reader, 1) + a.addAuxiliaryUsageAs(response, reader, 1, string(roles.RoleMarkReader)) said := strings.TrimSpace(response.Text()) if declaresNothingLeft(said) { // THE READER SAYING NOTHING IS LEFT IS A FACT, NOT A SILENCE. It used to diff --git a/internal/session/fuelrows_test.go b/internal/session/fuelrows_test.go new file mode 100644 index 0000000000..616416e82f --- /dev/null +++ b/internal/session/fuelrows_test.go @@ -0,0 +1,43 @@ +package session + +// THE FUEL TABLE HAS TO KNOW EVERY MODEL THIS BUILD SHIPS. The table is the +// fallback an orchestrated run meters against when no catalog reader is +// installed, so a shipped id with no row is a seat that quietly bills at the +// unpriced rate — the most expensive default seat in particular. This pins the +// five shipped defaults and every id in both crew tables to a real row, so a +// new default or a moved crew seat fails the build until the table is repaid. + +import ( + "testing" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/orchestrate" +) + +// TestEveryShippedModelHasAFuelRow: every default and every crew-table id +// resolves to a known fuel row rather than falling through to unpriced. +func TestEveryShippedModelHasAFuelRow(t *testing.T) { + shipped := []string{ + config.DefaultReflexModel, + config.DefaultLowModel, + config.DefaultWorkerModel, + config.DefaultHighModel, + config.DefaultMastermindModel, + } + for _, source := range config.CrewSources { + for _, preset := range config.CrewPresets { + models, ok := config.CrewModelsForSource(source, preset) + if !ok { + t.Fatalf("the %s crew has no %s preset", source, preset) + } + for _, model := range models { + shipped = append(shipped, model) + } + } + } + for _, id := range shipped { + if _, known := orchestrate.PriceOf(id); !known { + t.Errorf("no fuel row for shipped model %q", id) + } + } +} diff --git a/internal/session/groundladder.go b/internal/session/groundladder.go index cf16c1a929..c3cb24dc4f 100644 --- a/internal/session/groundladder.go +++ b/internal/session/groundladder.go @@ -1147,6 +1147,19 @@ func (t taskTree) carryBranchHomeLocked() (string, error) { // session's own and goes with the work it carried; what furrow is told is that // the record may be forgotten ([taskTree.dropUniverse]). func (t taskTree) releaseLanded() { + // THE SAME DOOR [taskTree.releaseKept] closes: a tree whose ground was never + // made has nowhere a worktree could be registered and no repository to ask, + // and a command with no directory runs in the process's own directory — so + // the removal below would take a working copy out of somebody else's + // checkout and the branch deletion would name a branch in it. + if strings.TrimSpace(t.root) == "" || strings.TrimSpace(t.dir) == "" { + return + } + // THE ROOT LOCK IS THE CALLER'S. Every road that reaches here is a landing + // that already holds it ([taskTree.carryBranchHome]'s span, task_run.go), + // the same way [taskTree.releaseKeptLocked] is reached, and the lock is not + // re-entrant: taking it again here waited on itself for the whole of a + // suite's timeout. if t.ownRepository() { _ = os.RemoveAll(t.dir) } else if _, err := git(t.root, "worktree", "remove", t.dir); err != nil { diff --git a/internal/session/guardian.go b/internal/session/guardian.go index d19619607b..dcc933e357 100644 --- a/internal/session/guardian.go +++ b/internal/session/guardian.go @@ -187,8 +187,9 @@ func (a *Agent) guardianAllows(ctx context.Context, hub *eventHub, call ai.ToolC } // The person pays for it, so it is folded into the session's auxiliary usage // — the same pocket the title and the summary come out of, and for the same - // reason it is not charged to the turn: no turn asked for it. - a.addAuxiliaryUsage(response, judge, 1) + // reason it is not charged to the turn: no turn asked for it. It names + // itself as a gate, so the row is seated by the gate's own tier. + a.addAuxiliaryUsageAs(response, judge, 1, string(roles.RoleGuardian)) if !guardianSaysAllow(response.Text()) { return false diff --git a/internal/session/harness_build.go b/internal/session/harness_build.go index 5339d79d6c..753524ee3b 100644 --- a/internal/session/harness_build.go +++ b/internal/session/harness_build.go @@ -1047,8 +1047,9 @@ func (a *Agent) harnessComplete(ctx context.Context, messages []ai.Message, mode return "", false, empty } // The design is spent on the person's account like every other auxiliary - // call (title.go, guardian.go): it is not a turn, and it is not free. - a.addAuxiliaryUsage(response, model, 1) + // call (title.go, guardian.go): it is not a turn, and it is not free. And + // it names itself a designer, so the row is seated by the designer's tier. + a.addAuxiliaryUsageAs(response, model, 1, string(roles.RoleDesigner)) // THE REPLY IS NOT THE ROOM'S HISTORY, and this is where it stopped being one. // It used to be journaled here verbatim, which meant a design reopened tomorrow // replayed the JSON envelope the designer had written — the same wall of braces diff --git a/internal/session/hermetic_checkout_test.go b/internal/session/hermetic_checkout_test.go index 3388179be8..b3f10ca26e 100644 --- a/internal/session/hermetic_checkout_test.go +++ b/internal/session/hermetic_checkout_test.go @@ -4,7 +4,9 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "sort" + "strconv" "strings" "testing" ) @@ -66,6 +68,15 @@ import ( // do not care what anything is called and still catch the commit and the files, // which is the shape that actually cost somebody an afternoon. // +// AND A LINKED WORKTREE OF A SHARED CLONE READS LESS STILL. Branches belong to +// the clone's common directory, not to the working copy they were read from, so +// from a linked worktree every sibling's task/* churn — a branch cut for their +// own run, a branch deleted when their run landed — would look exactly like +// this run's damage, and the branch reading is skipped outright there. The +// worktree reading narrows instead of skipping: registrations under this root +// can still be this suite's, and a sibling's registration elsewhere in the +// clone cannot. +// // It cannot see a content change to a file that was already dirty before the // run — hashing the whole checkout each run costs more than it is worth — and // the incident's shape, new commits, new files, new branches and new worktrees, @@ -73,9 +84,12 @@ import ( // checkout is what the tree looked like at a moment: the commit it stood on, // the paths git considered dirty or unknown, the branches it had, and the -// worktrees registered against it. +// worktrees registered against it. When the tree is a linked worktree of a +// shared clone, branches is nil — the reading is skipped, not empty — and the +// worktrees are only the ones registered under the tree itself. type checkout struct { root string + linked bool head string dirty map[string]bool branches map[string]bool @@ -99,12 +113,38 @@ func watchTheCheckout() checkout { if watched.head, err = gitLine(root, "rev-parse", "HEAD"); err != nil { return checkout{} } + watched.linked = isLinkedWorktree(root) + if !watched.linked { + watched.branches = branchNames(root) + } watched.dirty = porcelain(root) - watched.branches = branchNames(root) - watched.worktrees = worktreePaths(root) + watched.worktrees = worktreePaths(root, watched.linked) return watched } +// isLinkedWorktree says whether the working copy at root is one worktree among +// several of one shared clone, which `rev-parse` answers by naming two +// different directories: the repository this copy carries its state in, and +// the common one every sibling of the same clone points at. A plain checkout +// answers the same directory twice. +func isLinkedWorktree(root string) bool { + gitDir, err := gitLine(root, "rev-parse", "--git-dir") + if err != nil || gitDir == "" { + return false + } + commonDir, err := gitLine(root, "rev-parse", "--git-common-dir") + if err != nil || commonDir == "" { + return false + } + if !filepath.IsAbs(gitDir) { // a plain checkout answers relative to root + gitDir = filepath.Join(root, gitDir) + } + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(root, commonDir) + } + return filepath.Clean(gitDir) != filepath.Clean(commonDir) +} + // moved says how the checkout differs from what it was, and "" when it does // not. A watch that was never taken never reports. func (c checkout) moved() string { @@ -127,33 +167,40 @@ func (c checkout) moved() string { if moved := differing(c.dirty, porcelain(c.root)); len(moved) > 0 { said = append(said, "these paths are not what they were before the run:\n\t"+strings.Join(moved, "\n\t")) } - // A BRANCH IS A WRITE THAT MOVES NOTHING. `task/do-the-thing-` cut in - // the person's repository leaves HEAD exactly where it was and leaves the - // porcelain empty, and it is still the suite committing into a tree it does - // not own — the merge that would have moved HEAD is the only part that did - // not happen yet. - // - // AND A LIVE codeaf CREATES THE SAME BRANCHES. When another codeaf is using - // this checkout and TMPDIR is outside it, branch and worktree churn is - // ambient the way journal growth is (hermetic_test.go) — report the head - // and the porcelain, which a neighbour's ordinary work does not move, and - // leave the shared refs alone. The #578 TempDir-inside-checkout case still - // reports everything, because that is when the suite itself can mint them. - ignoreAmbientRefs := liveCodeafUsingCheckout(c.root) && !tempDirInsideCheckout(c.root) - if !ignoreAmbientRefs { - if moved := differing(c.branches, branchNames(c.root)); len(moved) > 0 { - said = append(said, "these branches are not what they were before the run:\n\t"+strings.Join(moved, "\n\t")) - } - // AND A WORKTREE IS A WRITE GIT IS TOLD TO IGNORE. The ground ladder puts a - // task's tree under `.codeaf/`, which .gitignore covers, so the porcelain - // above stays silent about a whole second checkout sitting in the person's - // tree. The registration is not ignorable: git keeps it, so it is asked for. - if moved := differing(c.worktrees, worktreePaths(c.root)); len(moved) > 0 { + // BRANCHES AND WORKTREES BELONG TO THE WHOLE SHARED CLONE the checkout was + // read from, and a LINKED worktree reads even less than the ambient case: + // every sibling of the same clone churns the same task/* refs this suite + // could not have written, so from a linked worktree the branch reading is + // skipped outright and the worktree reading keeps only the registrations + // under the checkout itself. The head and the porcelain above still watch + // everything, whatever the checkout is. + if c.linked { + if moved := differing(c.worktrees, worktreePaths(c.root, true)); len(moved) > 0 { said = append(said, "these worktrees are not what they were before the run:\n\t"+strings.Join(moved, "\n\t")) } - } else if moved := differing(c.branches, branchNames(c.root)); len(moved) > 0 || - len(differing(c.worktrees, worktreePaths(c.root))) > 0 { - fmt.Fprintf(os.Stderr, "checkout guard: another codeaf is using %s; ignoring task branch/worktree churn (head and porcelain still watched)\n", c.root) + } else { + // AND A LIVE codeaf CREATES THE SAME BRANCHES. When another codeaf is using + // this checkout and TMPDIR is outside it, branch and worktree churn is + // ambient the way journal growth is (hermetic_test.go) — report the head + // and the porcelain, which a neighbour's ordinary work does not move, and + // leave the shared refs alone. The #578 TempDir-inside-checkout case still + // reports everything, because that is when the suite itself can mint them. + ignoreAmbientRefs := liveCodeafUsingCheckout(c.root) && !tempDirInsideCheckout(c.root) + if !ignoreAmbientRefs { + if moved := differing(c.branches, branchNames(c.root)); len(moved) > 0 { + said = append(said, "these branches are not what they were before the run:\n\t"+strings.Join(moved, "\n\t")) + } + // AND A WORKTREE IS A WRITE GIT IS TOLD TO IGNORE. The ground ladder puts a + // task's tree under `.codeaf/`, which .gitignore covers, so the porcelain + // above stays silent about a whole second checkout sitting in the person's + // tree. The registration is not ignorable: git keeps it, so it is asked for. + if moved := differing(c.worktrees, worktreePaths(c.root, false)); len(moved) > 0 { + said = append(said, "these worktrees are not what they were before the run:\n\t"+strings.Join(moved, "\n\t")) + } + } else if moved := differing(c.branches, branchNames(c.root)); len(moved) > 0 || + len(differing(c.worktrees, worktreePaths(c.root, false))) > 0 { + fmt.Fprintf(os.Stderr, "checkout guard: another codeaf is using %s; ignoring task branch/worktree churn (head and porcelain still watched)\n", c.root) + } } if len(said) == 0 { return "" @@ -167,7 +214,7 @@ func (c checkout) moved() string { "`rev-parse --show-toplevel` answers with this checkout (task_run.go's repositoryRoot, " + "#578). Which one it was, this guard cannot tell you; if the run had GOTMPDIR or TMPDIR " + "pointing inside a checkout, suspect the second. Undo the above before pushing.\n" + - "(a second checkout running this same suite beside you cannot cause this; a test that names no directory can, and so can a temporary directory that is not outside your tree. A live codeaf using this checkout is ignored for branch/worktree churn when TMPDIR is outside the tree.)" + "(a second checkout running this same suite beside you cannot cause this; a test that names no directory can, and so can a temporary directory that is not outside your tree. A live codeaf using this checkout is ignored for branch/worktree churn when TMPDIR is outside the tree. When this checkout is a linked worktree of a shared clone, branch churn is not reported at all and the worktree reading covers only registrations under the checkout, because both belong to the clone the siblings share — a leak that only cut a task/ branch from inside a linked worktree slips past here, and the head and the porcelain above are what catch it.)" } // differing names every line that is in one listing and not the other, marked @@ -249,9 +296,12 @@ func harnessBranch(name string) bool { return strings.HasPrefix(name, harnessBranchPrefix) } -// worktreePaths is the set of worktree directories registered against root that -// THIS SUITE could have registered — including one it hid under an ignored -// folder, which is the whole reason this reading exists. +// worktreePaths is the set of worktree directories registered against root +// that THIS SUITE could have registered — including one it hid under an ignored +// folder, which is the whole reason this reading exists. For a linked worktree +// it is asked with underRoot, because the listing answers for the whole shared +// clone and only the registrations under the checkout itself can be the +// suite's. // // ONLY THE PATHS ARE KEPT. The plain listing carries each worktree's head // beside its path, so keeping the whole line would report a head move a second @@ -259,12 +309,22 @@ func harnessBranch(name string) bool { // question. The porcelain form is asked for so the path and the branch can be // read separately, and an unreadable tree is an empty set as everywhere else // here. -func worktreePaths(root string) map[string]bool { +func worktreePaths(root string, underRoot bool) map[string]bool { out, err := gitLines(root, "worktree", "list", "--porcelain") if err != nil { return map[string]bool{} } - return harnessWorktrees(root, out) + found := harnessWorktrees(root, out) + if !underRoot { + return found + } + under := strings.TrimSuffix(root, "/") + "/" + for path := range found { + if !strings.HasPrefix(path, under) { + delete(found, path) + } + } + return found } // harnessWorktrees reads `git worktree list --porcelain` — one record per @@ -440,58 +500,104 @@ func TestTheCheckoutGuardReportsOnlyWhatThisRunCouldHaveWritten(t *testing.T) { } } +// TestTheCheckoutGuardDoesNotReportASiblingWorktreesBranch is the shared-clone +// case, run for real: a linked worktree stands beside a sibling in one clone, +// the sibling cuts and takes away task/ branches and registers a worktree of +// its own while "the run" is watched, and none of it may read as this run's +// damage — branches belong to the clone's common directory, and the sibling's +// registrations stand outside the watched tree. The second half proves the +// churn is real: watched from the main checkout, where the refs are this +// checkout's own, the same branch deletion is reported. +func TestTheCheckoutGuardDoesNotReportASiblingWorktreesBranch(t *testing.T) { + shared := newTestRepo(t) + linked := filepath.Join(shared, "linked") + sibling := filepath.Join(shared, "sibling") + mustGit(t, shared, "worktree", "add", "-b", "task/this-run-0a1b2c", linked) + mustGit(t, shared, "worktree", "add", "-b", "task/sibling-run-9f8e7d", sibling) + // A branch the sibling will take away during the run, seen only through + // the clone's shared refs. + mustGit(t, shared, "branch", "task/sibling-gone-554433") + + t.Chdir(linked) + watched := watchTheCheckout() + if !watched.linked { + t.Fatal("the linked worktree was not read as linked, so this test is not standing where it means to") + } + // THE SAME REFS READ FROM THE MAIN CHECKOUT, before the churn, are this + // checkout's own — that watch is the contrast the narrowing has to hold + // against. + head, _ := gitLine(shared, "rev-parse", "HEAD") + watchedMain := checkout{root: shared, head: head, + dirty: porcelain(shared), branches: branchNames(shared), worktrees: worktreePaths(shared, false)} + + mustGit(t, sibling, "branch", "task/sibling-cuts-778899") + mustGit(t, sibling, "branch", "-D", "task/sibling-gone-554433") + mustGit(t, shared, "worktree", "add", "-b", "task/sibling-tree-112233", filepath.Join(shared, "sibling-tree")) + + if said := watched.moved(); said != "" { + t.Fatalf("a sibling worktree's churn was reported as this run's damage:\n%s", said) + } + + // AND THE SAME CHURN IS REPORTED FROM THE MAIN CHECKOUT — the narrowing + // above is tied to the linked worktree, not to the branch having been + // someone else's. + if said := watchedMain.moved(); !strings.Contains(said, "gone: task/sibling-gone-554433") { + t.Fatalf("from the main checkout the sibling's deleted branch was not reported:\n%s", said) + } +} + // liveCodeafUsingCheckout reports whether another codeaf process looks like it // is working in this repository right now — a chat, a tick, a hosted session — // whose task/ branches and `.codeaf/` worktrees would otherwise look exactly // like the suite's own leak. // -// IT IS BEST-EFFORT AND FAILS OPEN TO "NO". A machine without /proc, a binary +// THE QUESTION IS ASKED OF THE PROCESS TABLE, which exists wherever this +// program does: any other process whose command line names a worktree of the +// repository root belongs to is taken for a live codeaf. The worktrees come +// from `git worktree list --porcelain` — the checkout itself, and every tree +// registered against the same clone. A /proc walk was tried first and +// abandoned: it exists on Linux only, so on darwin the answer was always +// "no", and the ambient reading never engaged on the machine this suite is +// written on. +// +// IT IS BEST-EFFORT AND FAILS OPEN TO "NO". A machine without ps, a binary // renamed something else, or a sandbox that hides other processes simply keeps // the stricter reading; a false negative is a red that a person can re-run with // codeaf closed, and a false positive would hide a real suite write. func liveCodeafUsingCheckout(root string) bool { - under := strings.TrimSuffix(root, "/") + "/" - entries, err := os.ReadDir("/proc") + lines, err := gitLines(root, "worktree", "list", "--porcelain") if err != nil { return false } - self := os.Getpid() - for _, entry := range entries { - if !entry.IsDir() { - continue - } - name := entry.Name() - if len(name) == 0 || name[0] < '0' || name[0] > '9' { - continue - } - var pid int - for i := 0; i < len(name); i++ { - if name[i] < '0' || name[i] > '9' { - pid = -1 - break - } - pid = pid*10 + int(name[i]-'0') + places := map[string]bool{root: true} + for _, line := range lines { + if rest, ok := strings.CutPrefix(line, "worktree "); ok { + places[strings.TrimSpace(rest)] = true } - if pid <= 0 || pid == self { + } + out, err := exec.Command("ps", "-axo", "pid=,command=").Output() + if err != nil { + return false + } + self := os.Getpid() + for _, line := range strings.Split(string(out), "\n") { + pid, command, found := strings.Cut(strings.TrimSpace(line), " ") + if !found { continue } - cmdline, err := os.ReadFile("/proc/" + name + "/cmdline") - if err != nil { + // The suite's own test binary stands IN a worktree but does not name one + // on its command line, so it never reads as a neighbour here. + own, err := strconv.Atoi(strings.TrimSpace(pid)) + if err != nil || own <= 0 || own == self { continue } - joined := strings.ReplaceAll(string(cmdline), "\x00", " ") - if !strings.Contains(joined, "codeaf") { + if !strings.Contains(command, "codeaf") { continue } - // The suite's own test binary and helpers name codeaf too (`go test`, - // `codeaf-session-test-home`). A live chat is the binary itself, or a - // tick/host child, with the checkout in its cwd or its arguments. - cwd, err := os.Readlink("/proc/" + name + "/cwd") - if err == nil && (cwd == root || strings.HasPrefix(cwd, under)) { - return true - } - if strings.Contains(joined, root) { - return true + for place := range places { + if strings.Contains(command, place) { + return true + } } } return false diff --git a/internal/session/image.go b/internal/session/image.go index c7f36be81d..67248bc96c 100644 --- a/internal/session/image.go +++ b/internal/session/image.go @@ -493,7 +493,7 @@ func (a *Agent) runVision(ctx context.Context, hub *eventHub, live ai.Message, s provider.WithRole(ctx, lane.RoleTalk), callPurpose(roles.RoleVision), []ai.Message{live}, seer) if err == nil && response != nil { - a.addAuxiliaryUsage(response, seer, 1) + a.addAuxiliaryUsageAs(response, seer, 1, string(roles.RoleVision)) } answer := "" if response != nil { diff --git a/internal/session/loop.go b/internal/session/loop.go index 27a53fd9ef..6e5f84e3df 100644 --- a/internal/session/loop.go +++ b/internal/session/loop.go @@ -3866,7 +3866,7 @@ func (a *Agent) bank(call bankedCall) { } a.mu.Unlock() if call.ledger { - a.recordUsageLine(call.used, call.model, call.role, call.lane, call.reconciled) + a.recordUsageLine(call) } } diff --git a/internal/session/orchestrate.go b/internal/session/orchestrate.go index 1c459dd36f..94c49d9b38 100644 --- a/internal/session/orchestrate.go +++ b/internal/session/orchestrate.go @@ -142,6 +142,10 @@ func (a *Agent) RunOrchestrate(ctx context.Context, goal, model string, capDolla seq := a.orchestrateSeq named := strings.TrimSpace(model) source, session := a.config.RolesSource, a.model + // The published tariff rides with the rest of the config: read under the + // same lock every other config field on this path is, installed below + // beside the run that meters by it. + priceNow := a.config.ModelPrice // AND THE TANK IS HELD TO THIS SESSION'S OWN CAP. A session given a spend // rail may not hand out a run larger than the rail (rail.go's [Agent.railCap]); // it is read here, under the same lock every other config field on this path @@ -197,6 +201,22 @@ func (a *Agent) RunOrchestrate(ctx context.Context, goal, model string, capDolla // says which model is doing the work, and the answer must not be able to move // under a `/model` switch half way through the run. family.worker = worker.model() + // THE TANK ASKS A PUBLISHED TARIFF FIRST, and only falls back to the + // meter package's own table when nobody published one for the model — the + // same order of authority the reported cost on a call's usage already has + // over both. The catalog lives behind [Config.ModelPrice] because a meter + // must not import it: this function value is the whole seam, installed here + // because this is the one place an orchestrated run is built and a reader + // is already in hand. A nil reader installs nothing, and the table stands + // alone exactly as before. + // + // THE NIL CASE IS A SKIP, NOT AN INSTALL OF NOTHING. The seam is one + // package-level value shared by every run in the process, so passing a nil + // source through would not leave the table alone — it would pull whatever + // another session installed out from under a run still metering by it. + if priceNow != nil { + orchestrate.UsePrices(orchestrate.CatalogPrices(priceNow)) + } run := orchestrate.New(goal, planner, worker, orchestrate.Options{ Cap: capDollars, Lanes: orchestrateLanes, @@ -719,7 +739,7 @@ func (p *orchestratePlanner) ask(ctx context.Context, messages []ai.Message) (st if response == nil { return "", errors.New("the planner answered with nothing") } - p.agent.addAuxiliaryUsage(response, p.call.model, 1) + p.agent.addAuxiliaryUsageAs(response, p.call.model, 1, string(roles.RolePlanner)) p.orch.Charge(orchestrateCost(response, p.call.model)) return response.Text(), nil } diff --git a/internal/session/orchestrate_cost_test.go b/internal/session/orchestrate_cost_test.go index 163889fee5..c3d10ada15 100644 --- a/internal/session/orchestrate_cost_test.go +++ b/internal/session/orchestrate_cost_test.go @@ -13,7 +13,7 @@ func TestOrchestrateCostKeepsABareNameOutOfAVendorsRow(t *testing.T) { want float64 }{ {model: "deepseek-v4-flash", want: 1.25}, - {model: "deepseek/deepseek-v4-flash", want: 0.14}, + {model: "deepseek/deepseek-v4-flash-0731", want: 0.06}, } { if got := orchestrateCost(withoutReportedCost, test.model); got != test.want { t.Errorf("orchestrateCost without a reported cost for %q = %v, want %v", test.model, got, test.want) @@ -25,7 +25,7 @@ func TestOrchestrateCostKeepsABareNameOutOfAVendorsRow(t *testing.T) { PromptTokens: 1_000_000, Cost: &reportedCost, }} - for _, model := range []string{"deepseek-v4-flash", "deepseek/deepseek-v4-flash"} { + for _, model := range []string{"deepseek-v4-flash", "deepseek/deepseek-v4-flash-0731"} { if got := orchestrateCost(withReportedCost, model); got != reportedCost { t.Errorf("orchestrateCost with a reported cost for %q = %v, want %v", model, got, reportedCost) } diff --git a/internal/session/promptprofile.go b/internal/session/promptprofile.go index bc636c473d..607d000b44 100644 --- a/internal/session/promptprofile.go +++ b/internal/session/promptprofile.go @@ -29,8 +29,9 @@ package session // An earlier draft of this file had a second trigger — the crew's `worker` row, // on the reasoning that an open-weight model reasons like a worker whatever its // card claims. THAT WAS WRONG, and it is stated here so nobody restores it. -// `deepseek-v4-flash` and `glm-5.3-flash` are the open-weight models the frugal -// and balanced presets put in that seat, and served through a provider they have +// `glm-5.3-flash` and `glm-5.3` are the open-weight models the presets put in +// that seat — the first through balanced, the second at max — and served +// through a provider they have // a hundred and twenty-eight thousand tokens of room. That is a NORMAL SYSTEM and // it must get the full page: the seat rule would have taken sections off the page, // shelved `propose_task` and `tasks`, and turned saved memories OFF for anybody diff --git a/internal/session/promptprofile_test.go b/internal/session/promptprofile_test.go index 240f0c83a6..60b95d0cc7 100644 --- a/internal/session/promptprofile_test.go +++ b/internal/session/promptprofile_test.go @@ -75,8 +75,8 @@ func TestTheCatalogsWindowOutranksTheConfiguredOne(t *testing.T) { // THE CREW'S WORKER SEAT IS NOT A TRIGGER, AND A BIG WINDOW ON IT GETS THE WHOLE // PAGE. An earlier draft went lean whenever the conversation rode the crew's -// `worker` model, which is `deepseek/deepseek-v4-flash-0731` on the frugal preset -// and `z-ai/glm-5.3-flash` on the balanced one — models served with a hundred and +// `worker` model, which is `z-ai/glm-5.3-flash` through the balanced preset and +// `z-ai/glm-5.3` at max — models served with a hundred and // twenty-eight thousand tokens of room. That would have dropped sections, shelved // `propose_task` and turned saved memories off for anybody who picked a preset and // then chose that same model in chat, with nothing on screen saying so. Open diff --git a/internal/session/route_judge.go b/internal/session/route_judge.go index c94e87b599..0732b42b98 100644 --- a/internal/session/route_judge.go +++ b/internal/session/route_judge.go @@ -696,9 +696,10 @@ func (a *Agent) putRouteQuestion(ctx context.Context, role roles.Role, model, br if err != nil || response == nil { return routeVerdict{}, false } - // The person pays for it, out of the pocket every auxiliary call comes from, - // against the model that answered. - a.addAuxiliaryUsage(response, judge, 1) + // The person pays for it, out of the pocket every auxiliary call comes + // from, against the model that answered — and the call names itself on the + // row, so the ledger can seat it by the role's own tier. + a.addAuxiliaryUsageAs(response, judge, 1, string(role)) // The salvage ladder is internal/subharness's, shared rather than reimplemented // so that a fenced or smart-quoted reply is read here exactly as it is read diff --git a/internal/session/session.go b/internal/session/session.go index ca5b86efbb..a967cbf274 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -1077,6 +1077,29 @@ func (u Usage) CachedShare() (float64, bool) { // BaseURL are required. APIKey may be empty for a session opened before the // person has handed one over — the first-run setup's case — and every request // refuses until [Agent.SetAPIKey] lands it. +// TaskLanding is one landed node, handed whole to Config.TaskLanded. It is the +// record a caller outside this package needs to judge the work: what was asked, +// what came home, who ran it and who checked it, and what the run spent. The +// fields are copied from the node's own record at the moment the node reached a +// final state, so a reader that arrives late reads a fact rather than a +// half-open run. +type TaskLanding struct { + ID uint64 + State TaskState + Brief string + Deliverable string + Report string + Claim string + Ending string + Wrote []string + Changed int + Checks []string + Worker string + High string + CostUSD float64 + Tokens int +} + type Config struct { Workspace string // tools root here; all relative paths resolve inside it Model string @@ -1386,6 +1409,18 @@ type Config struct { // TaskProgressCheck is the test seam for leash checkpoints. Production uses // the node's ordinary read-only checker; a test may answer deterministically. TaskProgressCheck func(brief string, evidence []string) (working bool, reason string) + + // TaskLanded is called once per landed node, on its own goroutine, after the + // node's row is in the project's index (task_run.go's [Agent.reportTaskNode]). + // It carries [TaskLanding]: the node's record as the landing left it, the + // worker's model and the model the checking pass ran on (empty when there was + // none). A final state only — running and queued nodes land nothing — and a + // call that never blocks the reporting path: the reporting goroutine hands the + // landing over and moves on, and a caller that is slow holds up nothing but + // its own goroutine. Nil is off, which is what every caller that does not + // want the news hands in, and what this package then spends nothing on. + TaskLanded func(TaskLanding) + // TaskDeadline overrides one checkpoint interval. Zero keeps the one-hour // production interval and lets deadline behavior be tested without an hour. TaskDeadline time.Duration @@ -1810,6 +1845,20 @@ type Config struct { // (#941). A worker leaves it empty; the zero value is the node it always was. crewRole roles.Role + // repairRound marks this agent as ONE REPAIR ROUND'S FRESH WORKER + // (task_audit.go's [Agent.repairNode]) rather than the node's own. It + // exists for the usage ledger's seat ([Agent.agentKind]): a round is the + // node's escalation onto the careful tier, and a row that read it as the + // worker's seat would hide the cascade's whole bill inside the ordinary + // work's. It is set by the one builder that takes a model outright + // ([Agent.newTaskAgentOn]) and nowhere else. + // + // IT DOES NOT TOUCH [Config.crewRole] ON PURPOSE. A crew role is what this + // agent's calls are FOR in the router's vocabulary ([Agent.laneRole]), and + // a repair round is a leaf's turns of work in a worktree, not a one-answer + // gate — erranding its lane role would re-price every call it makes. + repairRound bool + // Errand marks this agent as the short exchange behind home's `ask here` // (cmd/codeaf's chatv3_exchange.go) rather than a conversation somebody // sits in. It is a conversation in every other way — a real model, a real diff --git a/internal/session/spellout.go b/internal/session/spellout.go index 02b14c18f3..ba44b978a3 100644 --- a/internal/session/spellout.go +++ b/internal/session/spellout.go @@ -178,8 +178,9 @@ func (a *Agent) SpellOut(ctx context.Context, draft string) string { return "" } // The person pays for it out of the same pocket the session's title, the - // namer and the shaper come out of, and no turn asked for it. - a.addAuxiliaryUsage(response, call.Model, 1) + // namer and the shaper come out of, and no turn asked for it. It names + // itself the spell-out, so the row is seated by the spell-out's tier. + a.addAuxiliaryUsageAs(response, call.Model, 1, string(roles.RoleSpellOut)) return cleanSpellOut(response.Text()) } diff --git a/internal/session/task_audit.go b/internal/session/task_audit.go index cf1cffebf7..edb9161f5e 100644 --- a/internal/session/task_audit.go +++ b/internal/session/task_audit.go @@ -1241,6 +1241,14 @@ func (a *Agent) auditOnce(ctx context.Context, node *TaskNode, tree taskTree, gr } defer func() { _ = auditor.Close() + // The model the check ran on is read beside the spend it folds in, for + // the same pocket's reason: a landing that later reports who checked the + // work names the model that actually answered, failover included, and + // not the one that was asked for. The auditor's model is read from the + // agent itself rather than from `on`, which is the asked-for id. + node.graph.mu.Lock() + node.checkedOn = auditor.Model() + node.graph.mu.Unlock() // The audit is part of what the node cost, so it lands in the same // pocket the node's own spend does (task_run.go's foldTaskUsage): the // person asked for a task, not for a task and separately for a judge. @@ -1572,7 +1580,7 @@ func (a *Agent) repairNode(ctx context.Context, node *TaskNode, tree taskTree, v // through the rounds this is, and what the check said that sent it back. defer a.enterPhase(node, taskBeatRepairing, round, a.config.TaskRepairRounds, taskFindingLine(verdict.evidence))() - child, err := a.newTaskAgentOn(ctx, taskGroundDir(node, tree), node, fmt.Sprintf("-repair%d", round), a.repairTierModel(node, lift)) + child, err := a.newTaskAgentOn(ctx, taskGroundDir(node, tree), node, fmt.Sprintf("-repair%d", round), a.repairTierModel(node, lift), true) if err != nil { fmt.Fprintf(log, "repair %d: could not start a worker: %v\n", round, err) return nil, "" diff --git a/internal/session/task_divide.go b/internal/session/task_divide.go index 68be2dbec0..5633a350b2 100644 --- a/internal/session/task_divide.go +++ b/internal/session/task_divide.go @@ -1607,9 +1607,10 @@ func (a *Agent) reviewDivision(ctx context.Context, parent *TaskNode, parsed div } return unanswered(why) } - // The person pays for it, out of the pocket every auxiliary call comes from, - // against the model that actually answered. - a.addAuxiliaryUsage(response, reviewer, 1) + // The person pays for it, out of the pocket every auxiliary call comes + // from, against the model that actually answered — and names itself on the + // row for the seat's sake. + a.addAuxiliaryUsageAs(response, reviewer, 1, string(roles.RoleDivision)) raw, err := subharness.Salvage(response.Text()) if err != nil { diff --git a/internal/session/task_landing_hook_test.go b/internal/session/task_landing_hook_test.go new file mode 100644 index 0000000000..c50d47efd1 --- /dev/null +++ b/internal/session/task_landing_hook_test.go @@ -0,0 +1,121 @@ +package session + +// THE LANDING HOOK: what a door outside this package reads when a task lands. +// +// A landed node hands its own record to Config.TaskLanded exactly once, on a +// goroutine of its own; a node still queued or running hands over nothing, and +// a door that wired no hook at all costs nothing. Every test here drives the +// real graph — admit, the frontier, [Agent.reportTaskNode] — and scripts only +// the work itself, which is the part a test may not have. + +import ( + "testing" + "time" +) + +// TestALandedTaskCallsTheLandingHookOnceWithTheRecordsFields pins the whole +// shape of one landing: the hook is called once, with the node's own record — +// the brief and the deliverable it was admitted under, the report it landed +// with, what it wrote and how many paths it changed, the checks it declared — +// beside the worker's model and the model its check ran on. +func TestALandedTaskCallsTheLandingHookOnceWithTheRecordsFields(t *testing.T) { + agent, _ := newTestAgent(t, &scriptedCompleter{}, nil) + landings := make(chan TaskLanding, 4) + agent.config.TaskLanded = func(landing TaskLanding) { landings <- landing } + + graph := agent.graph() + release := make(chan struct{}) + graph.run = func(node *TaskNode) { + <-release + node.graph.mu.Lock() + node.wrote = []string{"alpha.md"} + node.changed = []string{"alpha.md"} + node.Checks = []string{"go test ./..."} + node.checkedOn = "test/high" + node.graph.mu.Unlock() + node.finish("the report the node landed with", []string{"alpha.md"}, "", "") + node.graph.complete(node, TaskDone) + } + id := graph.reserve() + graph.admit(id, taskSpec{ + title: "read the law", + brief: "read ALPHA and say what it holds", + deliverable: "the reading", + acceptance: "a reader can check it", + model: "test/worker", + }) + node := graph.node(id) + + // THE NODE IS RUNNING FIRST, and a running node has no record to hand + // over: the start announcement travels the same reporting path and must + // reach the hook with nothing. + deadline := time.Now().Add(5 * time.Second) + for node.stateNow() != TaskRunning && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if node.stateNow() != TaskRunning { + t.Fatalf("the node never started running; it is %s", node.stateNow()) + } + select { + case landing := <-landings: + t.Fatalf("a running node called the landing hook with %+v", landing) + case <-time.After(50 * time.Millisecond): + } + + close(release) + waitDoneNode(t, node) + + select { + case landing := <-landings: + if landing.ID != id { + t.Fatalf("the landing names node %d, want %d", landing.ID, id) + } + if landing.State != TaskDone { + t.Fatalf("the landing's state is %s, want done", landing.State) + } + if landing.Brief != "read ALPHA and say what it holds" { + t.Fatalf("the landing's brief is %q", landing.Brief) + } + if landing.Deliverable != "the reading" { + t.Fatalf("the landing's deliverable is %q", landing.Deliverable) + } + if landing.Report != "the report the node landed with" { + t.Fatalf("the landing's report is %q", landing.Report) + } + if len(landing.Wrote) != 1 || landing.Wrote[0] != "alpha.md" { + t.Fatalf("the landing's wrote is %v", landing.Wrote) + } + if landing.Changed != 1 { + t.Fatalf("the landing's changed is %d, want 1", landing.Changed) + } + if len(landing.Checks) != 1 || landing.Checks[0] != "go test ./..." { + t.Fatalf("the landing's checks are %v", landing.Checks) + } + if landing.Worker != "test/worker" { + t.Fatalf("the landing's worker is %q, want test/worker", landing.Worker) + } + if landing.High != "test/high" { + t.Fatalf("the landing's high is %q, want test/high", landing.High) + } + case <-time.After(5 * time.Second): + t.Fatal("a landed task never called the landing hook") + } +} + +// TestANilLandingHookIsFine is the ordinary door: a session that wired no +// reader lands its nodes exactly as before, and nothing anywhere notices. +func TestANilLandingHookIsFine(t *testing.T) { + agent, _ := newTestAgent(t, &scriptedCompleter{}, nil) + if agent.config.TaskLanded != nil { + t.Fatal("a session built with no landing hook wired one anyway") + } + + graph := agent.graph() + graph.run = func(node *TaskNode) { + node.finish("done", nil, "", "") + node.graph.complete(node, TaskDone) + } + id := graph.reserve() + graph.admit(id, taskSpec{title: "a node with no reader", brief: "b", acceptance: "a"}) + waitDoneNode(t, graph.node(id)) +} diff --git a/internal/session/task_merge_round.go b/internal/session/task_merge_round.go index 24fd9d9304..0354fe32e0 100644 --- a/internal/session/task_merge_round.go +++ b/internal/session/task_merge_round.go @@ -345,7 +345,7 @@ func (a *Agent) undoMergeRound(node *TaskNode, tree taskTree, log io.Writer) { // the instruction — this one is not being told what is missing from the work, it // is being told that two people wrote the same lines. func (a *Agent) runResolver(ctx context.Context, node *TaskNode, tree taskTree, home string, files, changed []string, log io.Writer) []string { - child, err := a.newTaskAgentOn(ctx, tree.dir, node, "-resolve", "") + child, err := a.newTaskAgentOn(ctx, tree.dir, node, "-resolve", "", false) if err != nil { fmt.Fprintf(log, "merge round: could not start a worker — %v\n", err) return nil diff --git a/internal/session/task_person.go b/internal/session/task_person.go index 4799ee8855..43b24e941f 100644 --- a/internal/session/task_person.go +++ b/internal/session/task_person.go @@ -175,7 +175,7 @@ func (a *Agent) judgeDecomposable(ctx context.Context, brief string) (bool, []st if callErr != nil || response == nil { return false, nil, "" } - a.addAuxiliaryUsage(response, judge, 1) + a.addAuxiliaryUsageAs(response, judge, 1, string(roles.RolePlanner)) if verdict, ok := parseTaskJudge(response.Text()); ok { return verdict.Parallel, verdict.Parts, verdict.Why } diff --git a/internal/session/task_release_landed_test.go b/internal/session/task_release_landed_test.go new file mode 100644 index 0000000000..4a815758b6 --- /dev/null +++ b/internal/session/task_release_landed_test.go @@ -0,0 +1,69 @@ +package session + +import ( + "os" + "strings" + "testing" +) + +// A TREE WITH NO GROUND IS LEFT ALONE BY ITS RELEASE. +// +// [taskTree.releaseKept] refuses a tree with an empty root or directory, for +// the reason gitWith does: a command with no directory runs wherever the +// process happens to be, and a removal and a branch deletion pointed there +// take apart the checkout the process is standing in. [taskTree.releaseLanded] +// is the mirror image of releaseKept and was missing both the refusal and the +// root lock; these tests hold the refusal in place. + +// TestReleaseLandedLeavesAnEmptyTreeAlone proves the empty case never runs git +// at all: the tree is released while the process stands inside a real +// repository, and neither that repository's worktrees nor its branches move. +func TestReleaseLandedLeavesAnEmptyTreeAlone(t *testing.T) { + repo := newTestRepo(t) + for _, tree := range []taskTree{ + {}, // nothing was ever grounded + {dir: repo}, // a directory with no root + {root: repo}, // a root with no directory + {root: " ", dir: " ", branch: "task/x"}, // whitespace is emptiness too + } { + before := gitOut(t, repo, "worktree", "list", "--porcelain") + branches := gitOut(t, repo, "branch", "--list", "--format=%(refname:short)") + + t.Chdir(repo) + tree.releaseLanded() + + if after := gitOut(t, repo, "worktree", "list", "--porcelain"); after != before { + t.Errorf("releasing %+v changed the registered worktrees:\n%s", tree, after) + } + if after := gitOut(t, repo, "branch", "--list", "--format=%(refname:short)"); after != branches { + t.Errorf("releasing %+v changed the branches:\n%s", tree, after) + } + if _, err := os.Stat(repo); err != nil { + t.Fatalf("releasing %+v disturbed the repository: %v", tree, err) + } + } +} + +// TestReleaseLandedRemovesItsWorktreeAndBranch is the ordinary case beside it: +// a released landing still unregisters its worktree and deletes its branch, +// so the guard above cannot be read as the release having gone quiet. +func TestReleaseLandedRemovesItsWorktreeAndBranch(t *testing.T) { + repo := newTestRepo(t) + place := Place{Dir: t.TempDir(), Workspace: repo} + tree, err := prepareTaskTree(place, repo, "abcd1234abcd9876", 3, "paint the fence") + if err != nil { + t.Fatalf("prepareTaskTree: %v", err) + } + if !strings.Contains(gitOut(t, repo, "worktree", "list"), tree.dir) { + t.Fatalf("git does not know the worktree %s", tree.dir) + } + + tree.releaseLanded() + + if list := gitOut(t, repo, "worktree", "list"); strings.Contains(list, tree.dir) { + t.Errorf("the worktree is still registered:\n%s", list) + } + if branchIsThere(repo, tree.branch) { + t.Errorf("the branch %s is still there", tree.branch) + } +} diff --git a/internal/session/task_run.go b/internal/session/task_run.go index 0e71f27d15..d9be08e382 100644 --- a/internal/session/task_run.go +++ b/internal/session/task_run.go @@ -92,6 +92,7 @@ import ( "github.com/Agent-Field/codeaf/internal/approval" "github.com/Agent-Field/codeaf/internal/effort" "github.com/Agent-Field/codeaf/internal/exec/bare" + "github.com/Agent-Field/codeaf/internal/guard" "github.com/Agent-Field/codeaf/internal/provider" "github.com/Agent-Field/codeaf/internal/roles" ) @@ -648,6 +649,12 @@ type TaskNode struct { // reaches the project index (task_index.go) and the node's own notice on the // way past. cost float64 + // checkedOn is the model the node's checking pass ran on, read off the + // auditor when its spend folds in (task_audit.go) and empty when there was + // none — a node that never reached a check, or one whose check could not + // start. It is a fact about the last check, not a setting, and it is read by + // the landing hook ([Agent.reportTaskNode]) beside the worker's own model. + checkedOn string // input, output, cacheRead and cacheWrite are that same accumulation in // TOKENS, folded in beside the dollars and kept on the checkpoint with them. // @@ -3805,6 +3812,63 @@ func (n *TaskNode) burned() int { // ── the world hearing about a node ────────────────────────────────────────── +// tellTaskLanded hands one landed node to the session's own landing reader +// (Config.TaskLanded), when a door wired one. It runs on its own goroutine +// through guard.Go, so a reader that is slow — a call to a model outside the +// crew takes as long as it takes — holds up nothing on the reporting path, +// which is already answering to the note delivery beside it. +// +// The snapshot is taken here, on the reporting goroutine, under the node's own +// lock discipline (graph.mu, the room's spend read outside it as notice does): +// a landing built later on the reader's goroutine could read a node already +// re-armed, and the record the reader judged would not be the one that landed. +func (a *Agent) tellTaskLanded(node *TaskNode) { + hook := a.config.TaskLanded + if hook == nil { + return + } + // A landing is a final state only — work that ended on an answer: done, + // failed, or ended where nobody could check it. A node still queued or + // running has no record to judge yet, and its own updates already travelled + // the reporting path above. + switch node.stateNow() { + case TaskDone, TaskFailed, TaskUnverified: + default: + return + } + landing := node.landing() + guard.Go("task landed", func() { hook(landing) }) +} + +// landing copies the node out from under the graph's lock, shaped for a reader +// outside this package. The spend is read BEFORE the graph lock is taken, the +// same order [TaskNode.notice] keeps. +func (n *TaskNode) landing() TaskLanding { + tokens := n.burned() + n.graph.mu.Lock() + defer n.graph.mu.Unlock() + wrote := make([]string, len(n.wrote)) + copy(wrote, n.wrote) + checks := make([]string, len(n.Checks)) + copy(checks, n.Checks) + return TaskLanding{ + ID: n.id, + State: n.state, + Brief: n.spec.brief, + Deliverable: n.spec.deliverable, + Report: n.report, + Claim: n.claim, + Ending: string(n.endingLocked()), + Wrote: wrote, + Changed: len(n.changed), + Checks: checks, + Worker: n.spec.model, + High: n.checkedOn, + CostUSD: n.cost, + Tokens: tokens, + } +} + // reportTaskNode is the graph's report hook: one event for a surface, a row in // the project's index, and — on a final state — one note for the model. // @@ -3833,6 +3897,7 @@ func (a *Agent) reportTaskNode(node *TaskNode) { return } a.recordTaskIndex(node) + a.tellTaskLanded(node) transcript := taskURI(node.journalPath()) note := landingNoteLead(notice) + taskNote(notice, transcript, a.settlePolicy(), a.addressLanding(notice)) // WHETHER IT IS WORTH A TURN OF ITS OWN depends on whether anybody is waiting @@ -6657,15 +6722,15 @@ func worktreeDirt(dir string) string { return worktreeDirtIn(context.Background( // worktreeDirtIn is [worktreeDirt] under a context, for a reading somebody may // have to stop ([treeWatch.close]). func worktreeDirtIn(ctx context.Context, dir string) string { - args := []string{"-C", dir, "--no-optional-locks", "status", "--porcelain", "--untracked-files=all", "--", "."} + args := []string{"--no-optional-locks", "status", "--porcelain", "--untracked-files=all", "--", "."} for _, dropping := range taskDroppingNames() { args = append(args, ":(exclude)"+dropping) } - out, err := exec.CommandContext(ctx, "git", args...).Output() + out, err := gitContext(ctx, dir, nil, args...) if err != nil { return "" } - sum := sha256.Sum256(out) + sum := sha256.Sum256([]byte(out)) return string(sum[:8]) } @@ -7182,7 +7247,7 @@ func (a *Agent) spendLedger(node *TaskNode) *Agent { // has come back ([nodeMemory]). No store, no reflex, or a router that answered // nothing: the node opens with exactly the prompt it always did. func (a *Agent) newTaskAgent(ctx context.Context, dir string, node *TaskNode, suffix string) (*Agent, error) { - return a.newTaskAgentOn(ctx, dir, node, suffix, "") + return a.newTaskAgentOn(ctx, dir, node, suffix, "", false) } // newTaskAgentOn is [Agent.newTaskAgent] with the model said outright, and it @@ -7195,7 +7260,13 @@ func (a *Agent) newTaskAgent(ctx context.Context, dir string, node *TaskNode, su // is recorded as a bill rather than as a retarget — a repair round is one worker // among several a node takes, and a node whose row started naming the repair's // model would be telling a person their work moved when it did not. -func (a *Agent) newTaskAgentOn(ctx context.Context, dir string, node *TaskNode, suffix, on string) (*Agent, error) { +// AND `repair` IS THE CALLER SAYING WHICH KIND OF WORKER THIS IS, not something +// read off the model. It seats the round's usage rows high ([Agent.agentKind]), +// and it is stated rather than inferred from a named model because the two come +// apart: the cascade may floor to the model the work is already on, and a node +// that was admitted without an id has nothing to name at all — both of which +// would leave a round the ladder really did lift billing as ordinary work. +func (a *Agent) newTaskAgentOn(ctx context.Context, dir string, node *TaskNode, suffix, on string, repair bool) (*Agent, error) { // The model it is ACTUALLY on rather than the id it was admitted with, so a // second worker built for a node that was moved is built for where the node // now is ([TaskNode.runOn]). They are the same string for every node nothing @@ -7389,6 +7460,12 @@ func (a *Agent) newTaskAgentOn(ctx context.Context, dir string, node *TaskNode, // node hears about it while it happens: a card that would otherwise show // a task working says it is waiting instead. pacing: node.pacing, + // AND THE REPAIR ROUND SAYS SO, from the caller that knows it is one + // (task_audit.go's [Agent.repairNode]). The marker is what seats the + // round's rows high ([Agent.agentKind]); the crew role is deliberately + // left unset, because a round is a leaf's turns of work and erranding + // its lane role would re-price every call in it. + repairRound: repair, // AND THE NODE'S PULSE TRAVELS THE SAME WAY, for the same reason: the // checker and each repair round are the same NODE working, so a reader // outside the process must see one heartbeat across all of them rather @@ -9022,13 +9099,22 @@ func git(dir string, args ...string) (string, error) { return gitWith(dir, nil, args...) } -// gitWith is [git] with something extra in its environment, and it exists for -// exactly one caller: the ground ladder stages a parent's tree into AN INDEX OF -// ITS OWN so that handing out a child never moves the parent's index -// ([sealGroundWork]). GIT_INDEX_FILE is the only way to say that to git, and a +// gitWith is [gitContext] with [context.Background] in place of a caller's own +// context, kept as its own name because nearly every call here is that. The +// environment it takes exists for exactly one caller: the ground ladder stages +// a parent's tree into AN INDEX OF ITS OWN so that handing out a child never +// moves the parent's index ([sealGroundWork]). GIT_INDEX_FILE is the only way to say that to git, and a // second copy of the pager and editor settings beside it would be the drift the // one-source-of-truth law forbids. func gitWith(dir string, environment []string, args ...string) (string, error) { + return gitContext(context.Background(), dir, environment, args...) +} + +// gitContext is [gitWith] under a context, for a command somebody may have to +// stop before git answers ([worktreeDirtIn]'s watch). The refusal of an empty +// directory and the pinned environment live here, so there is one copy of +// each; every other spelling of a git call in this package is a wrapper of it. +func gitContext(ctx context.Context, dir string, environment []string, args ...string) (string, error) { // A COMMAND WITH NO DIRECTORY RUNS WHEREVER THE PROCESS HAPPENS TO BE. // // exec.Cmd reads an empty Dir as "the calling process's working directory", @@ -9047,7 +9133,7 @@ func gitWith(dir string, environment []string, args ...string) (string, error) { if strings.TrimSpace(dir) == "" { return "", errors.New("git: no directory to run in") } - command := exec.Command("git", args...) + command := exec.CommandContext(ctx, "git", args...) command.Dir = dir // A pager or an editor in the middle of a merge would hang a node forever on // a terminal it does not have. diff --git a/internal/session/task_shape.go b/internal/session/task_shape.go index 8239f6fbe9..ca67e68cd4 100644 --- a/internal/session/task_shape.go +++ b/internal/session/task_shape.go @@ -184,8 +184,9 @@ func (a *Agent) shapeBrief(ctx context.Context, request string) (shapedBrief, bo return shapedBrief{}, false } // The person pays for it out of the same pocket the title and the - // guardian come out of, and no turn asked for it. - a.addAuxiliaryUsage(response, call.Model, 1) + // guardian come out of, and no turn asked for it. The call names itself + // a shaper, so the row is seated by the shaper's own tier. + a.addAuxiliaryUsageAs(response, call.Model, 1, string(roles.RoleShaper)) if shaped, ok := parseShapedBrief(response.Text()); ok { return shaped, true } diff --git a/internal/session/taskgitdir_test.go b/internal/session/taskgitdir_test.go index 3ed374555a..296421cbcf 100644 --- a/internal/session/taskgitdir_test.go +++ b/internal/session/taskgitdir_test.go @@ -1,6 +1,7 @@ package session import ( + "context" "os" "path/filepath" "strings" @@ -17,6 +18,48 @@ import ( // wrote "task: Rewrite", "task: Measure" and "task: Paint" onto the branch of // the worktree it was launched from. Every call site names a directory, so one // that answers "" is a defect, and it fails here rather than somewhere else. +// TestStatusReadsWithNoDirectoryNeverRuns is the same proof for the two call +// sites that built `git -C ` themselves instead of asking [gitWith]: +// [worktreeDirtIn]'s dirty fingerprint and [UnsavedEditsNote]'s status reading. +// `git -C ""` is a no-op — git runs where the process is standing — so a direct +// exec.Command with an empty directory slipped past the refusal in gitWith. +// The repository the process stands in holds a TRACKED file with an unsaved +// edit, because [UnsavedEditsNote] reads with --untracked-files=no and an +// untracked file alone would leave it silent whether or not git ran. +func TestStatusReadsWithNoDirectoryNeverRuns(t *testing.T) { + repo := t.TempDir() + for _, args := range [][]string{{"init", "-q", "-b", "main"}, {"config", "user.name", "codeaf"}} { + if _, err := git(repo, args...); err != nil { + t.Skipf("no usable git here: %v", err) + } + } + writeFile(t, filepath.Join(repo, "note.md"), "work\n") + mustGit(t, repo, "add", "-A", "--", ".") + mustGit(t, repo, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "the note") + writeFile(t, filepath.Join(repo, "note.md"), "more work\n") + + // Both readings see the edit when they are told where to look; that is what + // makes their silence below a refusal rather than an empty status. + if got := UnsavedEditsNote(repo); got != unsavedEditsWord { + t.Fatalf("UnsavedEditsNote(repo) = %q, so the edit is not visible to a status read", got) + } + if got := worktreeDirtIn(context.Background(), repo); got == "" { + t.Fatal("worktreeDirtIn(repo) answered \"\", so the edit is not visible to a status read") + } + + t.Chdir(repo) + + if got := worktreeDirtIn(context.Background(), ""); got != "" { + t.Errorf("worktreeDirtIn with no directory returned %q, so a git command ran", got) + } + if got := worktreeDirt(""); got != "" { + t.Errorf("worktreeDirt with no directory returned %q, so a git command ran", got) + } + if got := UnsavedEditsNote(""); got != "" { + t.Errorf("UnsavedEditsNote with no directory returned %q, so a git command ran", got) + } +} + func TestAGitCommandWithNoDirectoryNeverRuns(t *testing.T) { // The proof that it never RAN, rather than merely never committed: this is // a real repository holding a real change, and a `git add` that reached it diff --git a/internal/session/taskpreflight.go b/internal/session/taskpreflight.go index 54a1098c74..0a500ea356 100644 --- a/internal/session/taskpreflight.go +++ b/internal/session/taskpreflight.go @@ -43,7 +43,6 @@ package session // an overlap worth a line ([hotFiles]). import ( - "os/exec" "path" "path/filepath" "strconv" @@ -153,8 +152,8 @@ func UnsavedEditsNote(root string) string { if strings.TrimSpace(root) == "" { return "" } - out, err := exec.Command("git", "-C", root, "status", "--porcelain", "--untracked-files=no").Output() - if err != nil || strings.TrimSpace(string(out)) == "" { + out, err := git(root, "status", "--porcelain", "--untracked-files=no") + if err != nil || strings.TrimSpace(out) == "" { return "" } return unsavedEditsWord diff --git a/internal/session/usage_ledger.go b/internal/session/usage_ledger.go index b3a2127f56..e95cf0f2c7 100644 --- a/internal/session/usage_ledger.go +++ b/internal/session/usage_ledger.go @@ -73,14 +73,17 @@ package session // turn forever. [UsageCache] keeps what it has parsed and reads only what // was appended since ([UsageCache.Read]). // -// WHAT IT DOES NOT HOLD, said plainly, because a page must not imply otherwise: -// the four-way token split (input and output are kept, the cache share is not — -// the journal this line's session id names has it), which of the five router -// slots a call ran under (nothing records that anywhere; `Role` here is -// internal/roles' auxiliary name, and only three auxiliary calls in the whole -// program name themselves at all), and any title for a task or a standing item — -// only their ids, which a page joins against the task index and the standing -// store it is already reading. +// WHAT IT HOLDS AND WHAT IT DOES NOT, said plainly, because a page must not +// imply otherwise. It now holds WHICH SEAT a call ran under — the tier's word +// ([Seat], usage_seat.go), with the registry's own name beside it where the +// call named one ([UsageLine.Role]) — so "did the money go on the seat that +// does the work or the seat that thinks" is answerable from this file alone. It +// still does not hold: the four-way token split (input and output are kept, the +// cache share is not — the journal this line's session id names has it), which +// of the five router slots a call ran under (nothing records that anywhere; the +// seat is the tier's chair and NOT the router's slot), and any title for a task +// or a standing item — only their ids, which a page joins against the task +// index and the standing store it is already reading. import ( "bufio" @@ -99,6 +102,7 @@ import ( "github.com/Agent-Field/agentfield/sdk/go/ai" "github.com/Agent-Field/codeaf/internal/home" "github.com/Agent-Field/codeaf/internal/provider" + "github.com/Agent-Field/codeaf/internal/roles" ) // UsageLedgerName is the file, under the v3 home directory @@ -139,14 +143,30 @@ type UsageLine struct { // provider knows, not a pretty name: a page that wants "opus 4.1" makes that // word itself, from one place, the way every other surface does. Model string `json:"model,omitempty"` - // Role is WHAT the call was for — "title", "taskname", "intake" — on the - // three auxiliary calls in the program that name themselves, and empty on - // every other line including every call a turn makes for itself. It is internal/roles' - // vocabulary and NOT the five router slots (execution, conversation, - // verification, naming, planning): nothing in the program records which slot - // a call ran under, and a page that labelled this column with those words - // would be inventing the join. + // Role is WHAT the call was for — "title", "guardian", "planner" — on + // every call that went through the role registry and named itself at the + // billing door, and empty on the rest, including every call a turn makes + // for itself. It is internal/roles' vocabulary and NOT the five router + // slots (execution, conversation, verification, naming, planning): nothing + // in the program records which slot a call ran under, and a page that + // labelled this column with those words would be inventing the join. Role string `json:"role,omitempty"` + // Seat is WHICH TIER'S MODEL ANSWERED, one word of the closed vocabulary + // [Seat] spells — reflex, low, worker, high, mastermind, judge, talk — derived at + // the bank door ([TagUsage], [SeatOfRole], [SeatOfAgent]) and never typed + // at a call site. It is the fact a spend page wants when it asks whether + // the money went on the seat that does the work or the seat that thinks, + // and it rides beside Role: a call the registry seated carries both, and + // the tier's word stays true of a row whose model id is a fallback, a pin + // or a rescue, because the seat names the chair the call ran in, not the + // id that answered. + // + // IT IS omitempty LIKE EVERY ADDITIVE FIELD BEFORE IT, and a row without + // one is a row nobody could seat: every row written before this field + // existed, an errand that resolved its model outside the registry (a media + // pin, a document reader, a tool ask), a receipt that arrived late. Empty + // reads as "nobody said", which is the truth about it. + Seat Seat `json:"seat,omitempty"` // Calls is how many provider requests this line covers, and it is ONE. The // field is kept rather than dropped because rows written before issue #269 // carry a whole turn's worth on one line — an observed `calls: 41` — and a @@ -800,7 +820,8 @@ func scanUsage(reader io.Reader, since time.Time) ([]UsageLine, int64, error) { // now happens on a writer goroutine ([RecordUsage]), the lock is still released // before the hand-off so no reader of the session's totals ever queues behind // the ledger at all. -func (a *Agent) recordUsageLine(used Usage, model, role string, lane laneFacts, reconciled bool) { +func (a *Agent) recordUsageLine(call bankedCall) { + used := call.used if used.Input == 0 && used.Output == 0 && used.CostUSD == 0 { return } @@ -815,14 +836,13 @@ func (a *Agent) recordUsageLine(used Usage, model, role string, lane laneFacts, line := UsageLine{ At: now, Day: now.Local().Format(usageDayLayout), - Model: strings.TrimSpace(model), - Role: strings.TrimSpace(role), + Model: strings.TrimSpace(call.model), Calls: used.Calls, Input: used.Input, Output: used.Output, USD: used.CostUSD, Empty: used.EmptyReflex > 0, - Reconciled: reconciled, + Reconciled: call.reconciled, Session: session, // The node this agent IS, and nothing for a conversation — the same @@ -837,6 +857,21 @@ func (a *Agent) recordUsageLine(used Usage, model, role string, lane laneFacts, Workspace: strings.TrimSpace(a.config.Workspace), } + // THE ROW'S TWO NAMES GO ON THROUGH ONE DOOR ([TagUsage]), and the seat + // is argued in only where it is known outright: an agent's own turns bill + // to the seat [SeatOfAgent] answers for the kind this agent is — talk on a + // conversation, worker on a node, high on the checker and a repair round — + // and every other row falls back to the role's own tier. That fallback is + // the errand's word when the call went through the registry and nobody's + // word at all when it resolved its model elsewhere (a media pin, a tool + // ask), which is exactly the row the emptiness law wants: as wide as it + // was, and saying "nobody said" where nobody did. + seat := Seat("") + if call.turn { + seat, _ = SeatOfAgent(a.agentKind()) + } + line = TagUsage(line, roles.Role(call.role), seat) + // THE LANE HALF OF THE LINE GOES ON THROUGH ONE DOOR, and it comes in as an // argument rather than being read from anywhere here — because the only // honest source for it is the request this row is about, and by the time a @@ -849,7 +884,7 @@ func (a *Agent) recordUsageLine(used Usage, model, role string, lane laneFacts, // Under the emptiness law an unwatched call therefore writes all five // absent, which is the true sentence "nobody said" rather than a zero // somebody reads as a figure. - RecordUsage(path, usageFromResponse(line, lane.Lane, lane.TTFT, lane.Gen, lane.Output, lane.Hedged, lane.Waste)) + RecordUsage(path, usageFromResponse(line, call.lane.Lane, call.lane.TTFT, call.lane.Gen, call.lane.Output, call.lane.Hedged, call.lane.Waste)) } // usageTaskID spells a node's id the way the task index spells it, and answers diff --git a/internal/session/usage_ledger_test.go b/internal/session/usage_ledger_test.go index 918fc3ff85..b7ceaeb8ee 100644 --- a/internal/session/usage_ledger_test.go +++ b/internal/session/usage_ledger_test.go @@ -468,7 +468,7 @@ func TestAConversationLineSpellsNoEmptyIds(t *testing.T) { if err != nil { t.Fatalf("marshal: %v", err) } - for _, absent := range []string{`"task"`, `"standing"`, `"workspace"`, `"role"`} { + for _, absent := range []string{`"task"`, `"standing"`, `"workspace"`, `"role", "seat"`} { if strings.Contains(string(line), absent) { t.Fatalf("an empty %s reached the file: %s", absent, line) } diff --git a/internal/session/usage_seat.go b/internal/session/usage_seat.go new file mode 100644 index 0000000000..a9711eafb3 --- /dev/null +++ b/internal/session/usage_seat.go @@ -0,0 +1,172 @@ +package session + +// THE SEAT: WHICH TIER'S MODEL ANSWERED, IN ONE WORD. +// +// A model id is an address (usage_ledger.go). The SEAT is the chair the call +// ran in — which class of model the person's settings put there. A spend page +// can already say what a day cost and which model took the money; without a +// seat it cannot say whether the money went on the seat that does the work or +// the seat that thinks. The vocabulary is internal/roles' five tiers — the +// classes the person actually configures — plus `judge`, the seat a run's +// judge is billed to, and `talk`, the one kind of call no tier governs: a +// conversation's own turns. +// +// THE WORDS ARE DERIVED, NEVER TYPED AT A CALL SITE. The registry already +// knows which tier a role sits on ([roles.TierOf]), so a call that went +// through it is seated by one hop, and the only hand-written table here is +// the five tiers to the five words ([SeatOfTier]). A seat is a fact about +// which tier's model answered, not a judgement about the call, and the seven +// words are the whole vocabulary: no free text ever reaches the field +// ([TagUsage] drops anything else). + +import ( + "strings" + + "github.com/Agent-Field/codeaf/internal/roles" +) + +// Seat is one word of the closed vocabulary a usage row's seat field carries. +type Seat string + +const ( + // SeatReflex is the cheapest of all: the two calls made every single turn. + SeatReflex Seat = "reflex" + // SeatLow is the cheap, fast one: a title, a caption, the guardian. + SeatLow Seat = "low" + // SeatWorker is the tier that DOES THE WORK: a task node's own turns. + SeatWorker Seat = "worker" + // SeatHigh is the capable, expensive one: the auditor, a repair round. + SeatHigh Seat = "high" + // SeatMastermind is the tier whose one answer shapes all the others: the + // planner, the designer, a division review. + SeatMastermind Seat = "mastermind" + // SeatJudge is the seat a run's judge is billed to, mapped from no tier + // and no role. + SeatJudge Seat = "judge" + // SeatTalk is a conversation's own turns — the one kind of call no tier + // governs, because the person is sitting in it and its model is theirs to + // change mid-sentence. + SeatTalk Seat = "talk" +) + +// Seats is the whole vocabulary, cheapest first, talk last. Talk closes the +// list rather than opening it because it is not in the tier economy at all: it +// is the word for the turns a conversation makes for itself, and a settings +// surface that drew it as a tier would be drawing a dial that answers nothing. +var Seats = []Seat{SeatReflex, SeatLow, SeatWorker, SeatHigh, SeatMastermind, SeatJudge, SeatTalk} + +// SeatValid reports whether a word is one of the seven. It compares EXACTLY: no +// trimming, no case folding. A word from outside the set is not a seat and is +// never written to a row — [TagUsage] drops it rather than writing a guess. +func SeatValid(word string) bool { + for _, seat := range Seats { + if string(seat) == word { + return true + } + } + return false +} + +// SeatOfTier maps each of [roles.Tiers] to its own word, and reports false for +// anything else. No tier maps to talk, because talk is what a conversation's +// own turns answer under and no tier governs those — and none maps to judge +// either, because judge is the seat a run's judge is billed to and no tier +// holds that. +func SeatOfTier(tier roles.Tier) (Seat, bool) { + switch tier { + case roles.TierReflex: + return SeatReflex, true + case roles.TierLow: + return SeatLow, true + case roles.TierWorker: + return SeatWorker, true + case roles.TierHigh: + return SeatHigh, true + case roles.TierMastermind: + return SeatMastermind, true + } + return "", false +} + +// SeatOfRole is the seat a registered role's calls bill to: [roles.TierOf] +// followed by [SeatOfTier], one hop through the registry rather than a second +// hand-written table. A role the registry never had is nobody's seat — false, +// and the row carries no word rather than a guess. +func SeatOfRole(role roles.Role) (Seat, bool) { + tier, ok := roles.TierOf(role) + if !ok { + return "", false + } + return SeatOfTier(tier) +} + +// AgentKind is what an agent IS, in the one question a seat needs answered: +// whose turns are these? +type AgentKind string + +const ( + // AgentChat is a conversation somebody sits in — and every agent that is + // neither a node, a checker nor a repair round, because that is what they + // all are: a session answering for itself. + AgentChat AgentKind = "chat" + // AgentTask is one task node's runner (task_run.go). + AgentTask AgentKind = "task" + // AgentAudit is the read-only judge a node's landing waits on + // (task_audit.go's [Agent.newAuditAgent]). + AgentAudit AgentKind = "audit" + // AgentRepair is one repair round's fresh worker + // (task_audit.go's [Agent.repairNode]). + AgentRepair AgentKind = "repair" +) + +// SeatOfAgent is the seat an agent's OWN turns bill to: chat is talk, task is +// worker, audit and repair are high. An unknown kind is false, and the row it +// would have carried stays wordless rather than guessed. +func SeatOfAgent(kind AgentKind) (Seat, bool) { + switch kind { + case AgentChat: + return SeatTalk, true + case AgentTask: + return SeatWorker, true + case AgentAudit, AgentRepair: + return SeatHigh, true + } + return "", false +} + +// agentKind is what THIS agent is, from the three facts its config carries: +// the crew role it answers for (the checker, [Config.crewRole]), the repair +// marker its builder set ([Config.repairRound]), the node flag — and, the +// default, none of those: a conversation, or an agent standing in for one. +func (a *Agent) agentKind() AgentKind { + switch { + case a.config.crewRole != "": + return AgentAudit + case a.config.repairRound: + return AgentRepair + case a.config.InTask: + return AgentTask + } + return AgentChat +} + +// TagUsage writes a row's two name fields, and is the ONE door they go +// through. Nothing else on the line is touched. +// +// A non-empty role is written to UsageLine.Role; an empty role leaves whatever +// was there. The seat argument wins when it is one of the seven words; a word +// outside the set is dropped rather than written, and the seat then falls back +// to the role's own tier ([SeatOfRole]); if that fails too, whatever seat the +// line already carried stays — which is what keeps a caller that knows nothing +// about a row from overwriting a word somebody else wrote onto it. +func TagUsage(line UsageLine, role roles.Role, seat Seat) UsageLine { + if name := strings.TrimSpace(string(role)); name != "" { + line.Role = name + } + if SeatValid(string(seat)) { + line.Seat = seat + } else if derived, ok := SeatOfRole(role); ok { + line.Seat = derived + } + return line +} diff --git a/internal/session/usage_seat_test.go b/internal/session/usage_seat_test.go new file mode 100644 index 0000000000..b7f608c782 --- /dev/null +++ b/internal/session/usage_seat_test.go @@ -0,0 +1,382 @@ +package session + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/roles" +) + +// ── the vocabulary ────────────────────────────────────────────────────────── + +func TestTheSeatVocabularyIsSevenWordsCheapestFirstTalkLast(t *testing.T) { + want := []Seat{SeatReflex, SeatLow, SeatWorker, SeatHigh, SeatMastermind, SeatJudge, SeatTalk} + if len(Seats) != len(want) { + t.Fatalf("the vocabulary holds %d words, want the seven: %v", len(Seats), Seats) + } + for i, seat := range want { + if Seats[i] != seat { + t.Fatalf("Seats[%d] is %q, want %q — cheapest first, talk last", i, Seats[i], seat) + } + } +} + +func TestSeatValidComparesExactly(t *testing.T) { + for _, seat := range Seats { + if !SeatValid(string(seat)) { + t.Fatalf("SeatValid(%q) is false, want true", seat) + } + } + for _, word := range []string{"", "Talk", "TALK", " talk", "talk ", "worker\n", "work", "leaf", "mastermind "} { + if SeatValid(word) { + t.Fatalf("SeatValid(%q) is true, want false — no trimming, no folding", word) + } + } +} + +func TestSeatOfTierMapsEveryTierAndNothingElse(t *testing.T) { + for _, tier := range roles.Tiers { + seat, ok := SeatOfTier(tier) + if !ok { + t.Fatalf("SeatOfTier(%q) is false — a tier of the registry has no seat word", tier) + } + if !SeatValid(string(seat)) { + t.Fatalf("SeatOfTier(%q) wrote %q, which is not one of the seven", tier, seat) + } + } + // Talk is the word for a conversation's own turns, and no tier governs + // those, so no tier answers talk. Judge is the seat a run's judge is + // billed to, and no tier holds that either. + for _, tier := range []roles.Tier{"", "talk", "judge", "speech", "execution"} { + if _, ok := SeatOfTier(tier); ok { + t.Fatalf("SeatOfTier(%q) answered, want false", tier) + } + } +} + +func TestSeatOfRoleIsOneHopThroughTheRegistry(t *testing.T) { + want := map[roles.Role]Seat{ + roles.RoleWorker: SeatWorker, + roles.RoleAuditor: SeatHigh, + roles.RoleRepair: SeatHigh, + roles.RolePlanner: SeatMastermind, + roles.RoleTitle: SeatLow, + roles.RoleReflex: SeatReflex, + roles.RoleHandoff: SeatMastermind, + roles.RoleGuardian: SeatLow, + roles.RoleCaption: SeatLow, + } + for role, seat := range want { + got, ok := SeatOfRole(role) + if !ok { + t.Fatalf("SeatOfRole(%q) is false — a registered role has no seat", role) + } + if got != seat { + t.Fatalf("SeatOfRole(%q) is %q, want %q", role, got, seat) + } + } + // The media roles are declared and deliberately never registered, so they + // seat nobody: false, and the row carries no word rather than a guess. + for _, role := range []roles.Role{roles.RoleSpeech, roles.RoleVideo, "hand", "nowhere"} { + if seat, ok := SeatOfRole(role); ok { + t.Fatalf("SeatOfRole(%q) answered %q, want false — a role the registry never had", role, seat) + } + } + // And no role of the registry answers judge: judge is the seat a run's + // judge is billed to, and the registry holds no role for it. + for role := range want { + if seat, ok := SeatOfRole(role); ok && seat == SeatJudge { + t.Fatalf("SeatOfRole(%q) answered judge, want a tier's word", role) + } + } +} + +func TestSeatOfAgentIsTheTurnsOwnSeat(t *testing.T) { + want := map[AgentKind]Seat{ + AgentChat: SeatTalk, + AgentTask: SeatWorker, + AgentAudit: SeatHigh, + AgentRepair: SeatHigh, + } + for kind, seat := range want { + got, ok := SeatOfAgent(kind) + if !ok { + t.Fatalf("SeatOfAgent(%q) is false, want %q", kind, seat) + } + if got != seat { + t.Fatalf("SeatOfAgent(%q) is %q, want %q", kind, got, seat) + } + } + for _, kind := range []AgentKind{"", "crew", "Talk", "subharness"} { + if seat, ok := SeatOfAgent(kind); ok { + t.Fatalf("SeatOfAgent(%q) answered %q, want false — an unknown kind seats nobody", kind, seat) + } + } +} + +func TestAgentKindReadsTheThreeConfigFacts(t *testing.T) { + want := []struct { + name string + more []func(*Config) + kind AgentKind + }{ + {"a conversation", nil, AgentChat}, + {"an errand beside one", []func(*Config){func(config *Config) { config.Errand = true }}, AgentChat}, + {"a task node", []func(*Config){func(config *Config) { config.InTask = true }}, AgentTask}, + {"the checker", []func(*Config){func(config *Config) { config.InTask = true; config.crewRole = roles.RoleAuditor }}, AgentAudit}, + {"a repair round", []func(*Config){func(config *Config) { config.InTask = true; config.repairRound = true }}, AgentRepair}, + } + for _, want := range want { + agent, _ := ledgerAgent(t, filepath.Join(t.TempDir(), UsageLedgerName), want.more...) + if got := agent.agentKind(); got != want.kind { + t.Fatalf("%s: agentKind() is %q, want %q", want.name, got, want.kind) + } + } +} + +// ── TagUsage ──────────────────────────────────────────────────────────────── + +func TestTagUsageIsTheOneDoorForARowsNames(t *testing.T) { + lines := []struct { + name string + line UsageLine + role roles.Role + seat Seat + want func(t *testing.T, got UsageLine) + }{ + { + name: "a clean role and the seat that goes with it", + line: UsageLine{Model: "opus"}, + role: roles.RoleTitle, seat: SeatLow, + want: func(t *testing.T, got UsageLine) { + if got.Role != "title" || got.Seat != SeatLow { + t.Fatalf("the row names %q seated %q, want title/low", got.Role, got.Seat) + } + }, + }, + { + name: "the seat argument wins even against the role's own tier", + line: UsageLine{Model: "opus"}, + role: roles.RolePlanner, seat: SeatWorker, + want: func(t *testing.T, got UsageLine) { + if got.Seat != SeatWorker { + t.Fatalf("the seat is %q, want worker — the argument wins", got.Seat) + } + }, + }, + { + name: "a word outside the set falls back to the role's own tier", + line: UsageLine{Model: "opus"}, + role: roles.RolePlanner, seat: Seat("leaf"), + want: func(t *testing.T, got UsageLine) { + if got.Seat != SeatMastermind { + t.Fatalf("the seat is %q, want mastermind — the role's own tier", got.Seat) + } + }, + }, + { + name: "an empty role leaves whatever was there", + line: UsageLine{Role: "caption", Seat: SeatLow}, + role: "", seat: Seat(""), + want: func(t *testing.T, got UsageLine) { + if got.Role != "caption" || got.Seat != SeatLow { + t.Fatalf("the row names %q seated %q, want caption/low untouched", got.Role, got.Seat) + } + }, + }, + { + name: "an unseatable call keeps the word it already carried", + line: UsageLine{Role: "hand", Seat: SeatHigh}, + role: "hand", seat: Seat("broad"), + want: func(t *testing.T, got UsageLine) { + if got.Role != "hand" || got.Seat != SeatHigh { + t.Fatalf("the row names %q seated %q, want hand/high kept", got.Role, got.Seat) + } + }, + }, + { + name: "nothing else on the line is touched", + line: UsageLine{Model: "opus", USD: 0.02, Input: 900, Output: 40, Lane: "Fast", TTFTms: 640}, + role: roles.RoleGuardian, seat: SeatLow, + want: func(t *testing.T, got UsageLine) { + if got.Model != "opus" || got.USD != 0.02 || got.Input != 900 || got.Output != 40 || got.Lane != "Fast" || got.TTFTms != 640 { + t.Fatalf("the row moved: %+v", got) + } + }, + }, + } + for _, want := range lines { + got := TagUsage(want.line, want.role, want.seat) + want.want(t, got) + } +} + +// ── the rows through the bank door ────────────────────────────────────────── + +// A conversation's own turns answer under talk, the one seat no tier governs. +func TestAConversationTurnsRowNamesItsSeatTalk(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + agent, _ := ledgerAgent(t, ledger) + + var turn Usage + bankCall(agent, &turn, "deepseek/deepseek-v4-flash", 900, 232, 0.31, laneFacts{}) + + row := sealRow(t, ledger) + wantRowFields(t, row, map[string]any{"seat": "talk"}) +} + +// A node's turns bill to the seat that does the work, and an errand the node +// runs beside that work bills to the errand's OWN tier — the seat names who +// answered, not who asked. +func TestATaskNodeSeatsItsTurnsWorkerAndItsErrandsByTheirRole(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + agent, _ := ledgerAgent(t, ledger, func(config *Config) { config.InTask = true }) + + var turn Usage + bankCall(agent, &turn, "deepseek/deepseek-v4-flash", 900, 232, 0.31, laneFacts{}) + agent.addAuxiliaryUsageAs(billedCall("haiku-4.5", 300, 12, 0.002), "haiku-4.5", 1, auxRoleHandoff) + + rows := rawUsageRows(t, ledger) + if len(rows) != 2 { + t.Fatalf("wrote %d rows, want the node's turn and the handoff errand: %v", len(rows), rows) + } + for _, row := range rows { + if row["role"] == auxRoleHandoff { + wantRowFields(t, row, map[string]any{"role": "handoff", "seat": "mastermind"}) + } else { + wantRowFields(t, row, map[string]any{"seat": "worker"}) + } + } +} + +// The checker is the node as far as everybody watching is concerned, and its +// bill is the gate's: high. +func TestTheAuditorsTurnsRowSeatsHigh(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + agent, _ := ledgerAgent(t, ledger, func(config *Config) { + config.InTask = true + config.crewRole = roles.RoleAuditor + }) + + var turn Usage + bankCall(agent, &turn, "deepseek/deepseek-v4-flash", 900, 232, 0.31, laneFacts{}) + + wantRowFields(t, sealRow(t, ledger), map[string]any{"seat": "high"}) +} + +// A repair round's fresh worker escalates onto the careful tier, and its rows +// say so rather than hiding the cascade's bill inside the ordinary work's. +func TestARepairRoundsRowSeatsHigh(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + agent, _ := ledgerAgent(t, ledger, func(config *Config) { + config.InTask = true + config.repairRound = true + }) + + var turn Usage + bankCall(agent, &turn, "deepseek/deepseek-v4-flash", 900, 232, 0.31, laneFacts{}) + + wantRowFields(t, sealRow(t, ledger), map[string]any{"seat": "high"}) +} + +// A named errand carries its registry name AND the tier's word that name sits +// on, whatever agent ran it. +func TestANamedErrandRowCarriesItsRoleAndItsTierSeat(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + agent, _ := ledgerAgent(t, ledger) + + agent.addAuxiliaryUsageAs(billedCall("haiku-4.5", 300, 12, 0.002), "haiku-4.5", 1, auxRoleTitle) + + rows := rawUsageRows(t, ledger) + if len(rows) != 1 { + t.Fatalf("wrote %d rows, want the one named errand: %v", len(rows), rows) + } + wantRowFields(t, rows[0], map[string]any{"role": "title", "seat": "low"}) +} + +// An errand whose word the registry never had carries the word and no seat: +// the word is still true, the tier is not. +func TestAnUnregisteredErrandWordCarriesItsRoleButNoSeat(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + agent, _ := ledgerAgent(t, ledger) + + agent.addAuxiliaryUsageAs(billedCall("haiku-4.5", 300, 12, 0.002), "haiku-4.5", 1, auxRoleHand) + + rows := rawUsageRows(t, ledger) + if len(rows) != 1 { + t.Fatalf("wrote %d rows, want the one worded errand: %v", len(rows), rows) + } + row := rows[0] + wantRowFields(t, row, map[string]any{"role": "hand"}) + wantRowSilentAbout(t, row, "seat") +} + +// An errand that resolved its model outside the registry — a media pin, a +// document reader, a tool ask — names nothing and seats nobody: the row is +// exactly as wide as it was. +func TestAnUnnamedErrandRowStaysSeatless(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + agent, _ := ledgerAgent(t, ledger) + + agent.addAuxiliaryUsage(billedCall("vendor/pin", 10, 900, 0.04), "vendor/pin", 1) + + row := sealRow(t, ledger) + wantRowSilentAbout(t, row, "role", "seat") +} + +// A receipt that arrived after the call names no role of its own, so it seats +// nobody either: the figures are true, the word was never in them. +func TestAReconciledReceiptStaysSeatless(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + agent, _ := ledgerAgent(t, ledger) + + agent.bank(bankedCall{ + used: Usage{Calls: 1, Input: 100, Output: 10, CostUSD: 0.02}, + model: "deepseek/deepseek-v4-flash", lane: laneFacts{}, + ledger: true, late: true, reconciled: true, + }) + + row := sealRow(t, ledger) + wantRowFields(t, row, map[string]any{"reconciled": true}) + wantRowSilentAbout(t, row, "role", "seat") +} + +// EVERY ROW ALREADY IN THE FILE STILL DECODES: an old row reads back with an +// empty seat and an empty role, which is the truth about it. +func TestAnOldRowStillDecodesWithoutASeat(t *testing.T) { + ledger := filepath.Join(t.TempDir(), UsageLedgerName) + old := `{"at":"2026-01-05T09:00:00Z","day":"2026-01-05","model":"opus","calls":1,"in":10,"out":2,"usd":0.01}` + if err := os.WriteFile(ledger, []byte(old+"\n"), 0o600); err != nil { + t.Fatalf("write the old ledger: %v", err) + } + lines, err := ReadUsage(ledger, time.Time{}) + if err != nil || len(lines) != 1 { + t.Fatalf("read %d lines, %v", len(lines), err) + } + if lines[0].Seat != "" || lines[0].Role != "" || lines[0].Model != "opus" { + t.Fatalf("the old row read back as %+v, want seatless and roleless", lines[0]) + } +} + +// The field is spelled `seat`, and a row nobody could seat is exactly as wide +// as it was. +func TestTheSeatIsSpelledSeatAndOmittedWhenUnset(t *testing.T) { + seated, err := json.Marshal(UsageLine{At: time.Now(), Day: "2026-09-17", Model: "m", Calls: 1, Input: 5, USD: 0.01, Session: "abc", Seat: SeatWorker}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(seated), `"seat":"worker"`) { + t.Fatalf("a seated row carries no seat word: %s", seated) + } + wide, err := json.Marshal(UsageLine{At: time.Now(), Day: "2026-09-17", Model: "m", Calls: 1, Input: 5, USD: 0.01, Session: "abc"}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if strings.Contains(string(wide), `"seat"`) { + t.Fatalf("an unseated row grew a seat key: %s", wide) + } +} diff --git a/internal/store/gate.go b/internal/store/gate.go index 1ef12e2310..ba1adbee15 100644 --- a/internal/store/gate.go +++ b/internal/store/gate.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "time" ) // DeliveryGate is the final judge's evidence about one job. Pass is the first @@ -441,6 +442,36 @@ func (g DeliveryGate) Whole() bool { return g.Pass || g.PolishClosed || g.Overturned } +// Done is whether this gate's own row reports the requested work as done: the +// judgement passed, or its only shortfall was behaviours the request states +// that no check exercises or asserts — the work is there, the checks for it are +// not. +// +// It is the positive fact a delivery gate had no way to state before, and two +// readers turn on it rather than on their own reading of the fields. The +// envelope's `core_done_seconds` is the moment it first became true for a run, +// and the share-of-spend rail measures growth from the same moment — one +// definition, because a rail and a receipt that disagreed about when the work +// was done would be two answers to one question. +func (g DeliveryGate) Done() bool { + if g.Pass { + return true + } + // A COVERAGE FINDING BESIDE ANOTHER SHORTFALL IS NOT DONE. The work is + // there only when the checks for it are the whole of what is missing; a + // gate that also names a consumer left unbound, a check of the work's own + // that fails, a constraint unmet or a mechanical failure has found the work + // itself short, and anchoring "done" there would measure growth from before + // the work was finished. + if g.Mechanical || g.Unreadable || g.Unjudged { + return false + } + if len(g.Consumers) > 0 || len(g.Unbound) > 0 || len(g.OwnFailing) > 0 || len(g.Constraint) > 0 { + return false + } + return len(g.Unexercised) > 0 || len(g.Unasserted) > 0 +} + // Cited is the gate's citations however they were written down. A row recorded // before the list existed carries only the joined line, and reading it as one // citation is the honest reading of it: that is exactly what it was when it was @@ -657,6 +688,59 @@ func (s *Store) DeliveryGateLineage(baseID string) ([]DeliveryGate, error) { return gates, nil } +// DeliveryGateAnchor returns where in the journal — and when — a job's +// requested work was FIRST found done: the oldest gate recorded for the node +// and everything spliced beneath its id whose verdict passed or left only a +// coverage finding. ok is false while no such gate has been recorded, which is +// a job whose core work has not been found done. +// +// The SEQUENCE is the point and not the payload. A spend rail measured from the +// moment the work was done reads the journal by sequence (SpendSinceSeq), and a +// caller given the payload alone would have to walk the whole event log to get +// back to the row it came from. +func (s *Store) DeliveryGateAnchor(baseID string) (seq int64, at time.Time, ok bool, err error) { + baseID = strings.TrimSpace(baseID) + if baseID == "" { + return 0, time.Time{}, false, nil + } + namespace := baseID + SplitNamespace + ceiling, ordered := idPrefixCeiling(namespace) + if !ordered { + return 0, time.Time{}, false, fmt.Errorf("read delivery gate anchor %q: %w: prefix has no ordered ceiling", baseID, ErrInvalid) + } + rows, err := s.db.Query(` + SELECT seq, ts, payload FROM events + WHERE kind = ? AND (node_id = ? OR (node_id >= ? AND node_id < ?)) + ORDER BY seq`, EventDeliveryGate, baseID, namespace, ceiling) + if err != nil { + return 0, time.Time{}, false, fmt.Errorf("read delivery gate anchor %q: %w", baseID, err) + } + defer rows.Close() + for rows.Next() { + var rowSeq int64 + var stamp, payload string + if err := rows.Scan(&rowSeq, &stamp, &payload); err != nil { + return 0, time.Time{}, false, fmt.Errorf("read delivery gate anchor %q: %w", baseID, err) + } + var gate DeliveryGate + if err := json.Unmarshal([]byte(payload), &gate); err != nil { + return 0, time.Time{}, false, fmt.Errorf("read delivery gate anchor %q: %w", baseID, err) + } + if !gate.Done() { + continue + } + when, err := parseTime(stamp) + if err != nil { + return 0, time.Time{}, false, fmt.Errorf("read delivery gate anchor %q: %w", baseID, err) + } + return rowSeq, when, true, nil + } + if err := rows.Err(); err != nil { + return 0, time.Time{}, false, fmt.Errorf("read delivery gate anchor %q: %w", baseID, err) + } + return 0, time.Time{}, false, nil +} + // EventAcceptance is the acceptance checklist journaled against the piece of // work it will be used to judge: the behaviours the person's own request states, // read from the request before any work began. diff --git a/internal/store/gate_done_test.go b/internal/store/gate_done_test.go new file mode 100644 index 0000000000..ea1b2826a8 --- /dev/null +++ b/internal/store/gate_done_test.go @@ -0,0 +1,28 @@ +package store + +import "testing" + +// A GATE IS DONE ON A PASS, OR WHEN THE CHECKS ARE ALL THAT IS MISSING. A +// coverage finding that stands beside any other shortfall has found the work +// itself short, and the moment it anchors would come before the work was +// finished. +func TestAGateIsDoneOnlyWhenTheChecksAreAllThatIsMissing(t *testing.T) { + for _, c := range []struct { + name string + gate DeliveryGate + want bool + }{ + {"a pass", DeliveryGate{Pass: true}, true}, + {"an unexercised finding alone", DeliveryGate{Unexercised: []string{"the flag turns on"}}, true}, + {"an unasserted finding alone", DeliveryGate{Unasserted: []string{"the flag turns off"}}, true}, + {"nothing found at all", DeliveryGate{}, false}, + {"a coverage finding beside a failing check of its own", DeliveryGate{Unexercised: []string{"a"}, OwnFailing: []string{"TestX"}}, false}, + {"a coverage finding beside an unbound consumer", DeliveryGate{Unexercised: []string{"a"}, Unbound: []string{"caller"}}, false}, + {"a coverage finding beside a constraint", DeliveryGate{Unasserted: []string{"a"}, Constraint: []string{"no new dependency"}}, false}, + {"a coverage finding on a mechanical failure", DeliveryGate{Unexercised: []string{"a"}, Mechanical: true}, false}, + } { + if got := c.gate.Done(); got != c.want { + t.Errorf("%s: Done() = %t, want %t", c.name, got, c.want) + } + } +} diff --git a/internal/store/lifecycle.go b/internal/store/lifecycle.go index 8f254d56cf..e15e0c7d56 100644 --- a/internal/store/lifecycle.go +++ b/internal/store/lifecycle.go @@ -2,6 +2,7 @@ package store import ( "database/sql" + "encoding/json" "errors" "fmt" "math" @@ -403,6 +404,80 @@ func (s *Store) release(claim Claim, reason string, recorded int) error { return nil } +// ReleasedTurnsFor is how many recorded turns the newest hand-on release left +// waiting for the next claim, or that no release of this node ever handed work +// on. It exists because the release is where the count the attempt before +// banked is written down — the settle that re-dispatched the node journaled it +// in the same breath as the reason — and the settle that has to COMPARE against +// it (a re-dispatch deciding whether it moved) cannot compose that comparison +// from its own memory across processes. +// +// Releases that took a claim back without handing work on are not it: they say +// nothing about how much the attempt before banked, and the newest count that +// does say is the one a re-dispatch is compared against. +func (s *Store) ReleasedTurnsFor(nodeID string) (int, bool, error) { + rows, err := s.db.Query(` + SELECT payload FROM events + WHERE node_id = ? AND kind = ? + ORDER BY seq DESC`, strings.TrimSpace(nodeID), EventNodeReleased) + if err != nil { + return 0, false, fmt.Errorf("read the handed-on turns: %w", err) + } + defer rows.Close() + for rows.Next() { + var payload string + if err := rows.Scan(&payload); err != nil { + return 0, false, fmt.Errorf("read the handed-on turns: %w", err) + } + var release releasePayload + if err := json.Unmarshal([]byte(payload), &release); err != nil { + return 0, false, fmt.Errorf("read the handed-on turns: %w", err) + } + if release.Recorded > 0 { + return release.Recorded, true, nil + } + } + if err := rows.Err(); err != nil { + return 0, false, fmt.Errorf("read the handed-on turns: %w", err) + } + return 0, false, nil +} + +// NodeRedispatches is how many times this node has been re-dispatched in place +// after running out of the room it was granted: the count of its releases that +// handed recorded turns on. It is read from the journal rather than counted +// anywhere in a process for the same reason every run figure is — a durable +// store outlives the binary that wrote it, and a re-dispatch a resident made is +// as much a re-dispatch as one this process made. +func (s *Store) NodeRedispatches(nodeID string) (int, error) { + rows, err := s.db.Query(` + SELECT payload FROM events + WHERE node_id = ? AND kind = ? + ORDER BY seq`, strings.TrimSpace(nodeID), EventNodeReleased) + if err != nil { + return 0, fmt.Errorf("read the re-dispatches: %w", err) + } + defer rows.Close() + redispatches := 0 + for rows.Next() { + var payload string + if err := rows.Scan(&payload); err != nil { + return 0, fmt.Errorf("read the re-dispatches: %w", err) + } + var release releasePayload + if err := json.Unmarshal([]byte(payload), &release); err != nil { + return 0, fmt.Errorf("read the re-dispatches: %w", err) + } + if release.Recorded > 0 { + redispatches++ + } + } + if err := rows.Err(); err != nil { + return 0, fmt.Errorf("read the re-dispatches: %w", err) + } + return redispatches, nil +} + // ── the claim reaper reads EVIDENCE OF LIFE, never a clock ────────────────── // // THE DEFECT THIS ANSWERS, and it is the same defect twice. This sweep used to diff --git a/internal/tui3/crew.go b/internal/tui3/crew.go index aefbeca475..f9c63cfca0 100644 --- a/internal/tui3/crew.go +++ b/internal/tui3/crew.go @@ -38,21 +38,45 @@ func (a *app) runCrew(arg string) { arg = strings.ToLower(strings.TrimSpace(arg)) if arg == "" { a.closeLists() - a.crewPick.start(config.CrewAt(a.profileDir), config.CrewSourceAt(a.profileDir), a.crewInheritedLine()) + a.crewPick.start(config.CrewAt(a.profileDir), config.CrewSourceAt(a.profileDir), + a.crewInheritedLine(), config.CrewPickAt(a.profileDir)) a.touch() return } + // THE THREE PICK WORDS, beside the presets: `/crew learn` says where the + // seats come from without moving any of them. The row is written alone and + // the confirmation is [app.crewApplied]'s, which names the pick beside the + // preset ([config.CrewSummaryPick]) — the same word the segment and + // /status now carry, so the answer and the frame cannot drift apart. + for _, pick := range config.CrewPicks { + if arg != pick { + continue + } + if err := config.SetCrewPick(a.profileDir, pick); err != nil { + a.note("could not set the pick · " + err.Error()) + return + } + a.crewApplied() + return + } if _, ok := config.CrewModels(arg); !ok { - // AN UNKNOWN WORD CHANGES NOTHING AND SHOWS THE THREE. A refusal that - // only said "no" would leave a person guessing at a word they were one - // letter away from, and the listing is the answer to the question they - // were really asking. - // The listing is a two-column page — a class word, then the model it would - // set — so its right-hand column is what THE PAYLOAD RULE lifts, read back - // off the text this call just built (payload.go's [columnFacts]). + // AN UNKNOWN WORD CHANGES NOTHING AND SAYS THE SIX, the shape every + // choice this surface refuses takes (effortchip.go's [app.runEffort]): + // a refusal that only said "no" would leave a person guessing at a word + // they were one letter away from, and the words are the answer to the + // question they were really asking. The listing is kept under them, + // because it is the comparison between the presets the words set. + // The listing is a two-column page — a class word, then the model it + // would set — so its right-hand column is what THE PAYLOAD RULE lifts, + // read back off the text this call just built (payload.go's + // [columnFacts]). + words := make([]string, 0, len(config.CrewPresets)+len(config.CrewPicks)) + words = append(words, config.CrewPresets...) + words = append(words, config.CrewPicks...) listing := a.crewListing() - a.noteFacts("/crew "+arg+" · not one of the three\n\n"+listing, - columnFacts(listing, false)...) + facts := append(words, columnFacts(listing, false)...) + a.noteFacts("/crew "+arg+" · not a crew word · "+strings.Join(words, " · ")+"\n\n"+listing, + facts...) return } a.applyCrew(arg) @@ -99,7 +123,12 @@ func (a *app) crewApplied() { if id := a.talkingTo(); id != "" { facts = append(facts, id) } - a.noteFacts(config.CrewSummary(a.profileDir)+" · "+a.crewUnchangedClause(), facts...) + // THE SUMMARY IS THE PICK-AWARE ONE ([config.CrewSummaryPick]), so a crew + // applied under a pick of catalog or learn confirms with the pick named + // beside the preset — the half the frame's segment and /status now carry. + // At the default pick it is the line the command has always confirmed + // with. + a.noteFacts(config.CrewSummaryPick(a.profileDir)+" · "+a.crewUnchangedClause(), facts...) } // applyCrewUnder is the chooser's enter: the family and the preset are ONE @@ -194,8 +223,9 @@ func (a *app) crewSegment() string { // ── the one reading of the four rows ──────────────────────────────────────── // crewReading is the profile's crew as this surface last read it: the preset -// word [config.CrewAt] derives from the live rows, the three class names beside -// it, and the settings generation the pair was read at. +// word [config.CrewAt] derives from the live rows, the pick row's word +// ([config.CrewPickAt]) when it is not the default one, the three class names +// beside it, and the settings generation the pair was read at. type crewReading struct { // word and segment are the two readings BUILT AT THE READING and not at the // draw. The status line asks for the segment on every frame, and a surface @@ -211,6 +241,11 @@ type crewReading struct { clause string preset string classes string + // pick is the pick row's word as the reading read it, and is the default + // word rather than empty — a surface that wants to know whether the pick + // is off the table compares it with [config.CrewPickTable], the way the + // word and segment above were built. + pick string // dir is the profile the pair was read from, so a surface handed a // different one answers from that one and not from a snapshot of the last. dir string @@ -280,12 +315,27 @@ func (a *app) crewReading() (crewReading, bool) { } if generation := config.SettingsGeneration(); !a.crew.taken || a.crew.generation != generation || a.crew.dir != a.profileDir { preset, classes := config.CrewAt(a.profileDir), config.CrewClasses(a.profileDir) + pick := config.CrewPickAt(a.profileDir) + // THE PICK RIDES THE WORD WHEN IT IS NOT THE DEFAULT ONE, in both the + // page's word and the status line's segment: `balanced · learn` says + // the budget and where the models for it come from in one word, and a + // segment that said only `balanced` under a learn pick would be the + // half-answer the crew segment exists to prevent. At the default pick + // the word is exactly what it has always been — the table is where the + // seats have always come from, and a word that named it would say + // something a person did not choose. + word, segment := preset+" · "+classes, "crew "+preset + if pick != config.CrewPickTable { + word = preset + " · " + pick + " · " + classes + segment = "crew " + preset + " · " + pick + } a.crew = crewReading{ - word: preset + " · " + classes, - segment: "crew " + preset, + word: word, + segment: segment, clause: preset + " crew", preset: preset, classes: classes, + pick: pick, dir: a.profileDir, generation: generation, taken: true, @@ -440,13 +490,19 @@ type crewPicker struct { // file, so a picker that asked the profile per frame would put a disk read // on the frame clock for a fact that cannot move while the list is up. inherited string + // pick is the pick row's word as the door read it, and the default word + // rather than empty. IT IS NOT STAGED: the family below is staged and + // enter writes it, but a pick word typed at the door is written at once + // ([app.runCrew]), so the pick on screen is always the pick in force — a + // reading, and not a row the list's keys could move. + pick string } -func (p *crewPicker) start(current, source, inherited string) { +func (p *crewPicker) start(current, source, inherited, pick string) { // source is already a word this build knows: every caller hands it // [config.CrewSourceAt]'s answer, which folds blank, unknown and retired words - // back to open. The fold lives there and nowhere else. - *p = crewPicker{open: true, current: current, persistedSource: source, inherited: inherited, source: source} + // back to the default family. The fold lives there and nowhere else. + *p = crewPicker{open: true, current: current, persistedSource: source, inherited: inherited, source: source, pick: pick} for i, preset := range config.CrewPresets { if preset == current { p.cursor = i @@ -489,6 +545,12 @@ const ( // crewCustomLine is the fourth reading, said as a fact about where the person // is rather than as a fourth row they could pick. crewCustomLine = "yours is none of the three — picking one puts all five back" + // crewPickLead is the pick's lead in the chooser — `picked from learn` — + // the same words the crew word and the segment ride when the pick is off + // its default ([app.crewReading]). It is said as a reading and not a row: + // the pick is written by a word at the door, not by anything this list's + // keys move. + crewPickLead = "picked from " // crewInheritedLead and crewInheritedTail wrap the row a profile older than // a seat never wrote: // @@ -515,6 +577,9 @@ func (p *crewPicker) height() int { if p.current == config.CrewCustom { height++ } + if p.pick != "" && p.pick != config.CrewPickTable { + height++ + } if p.inherited != "" { height++ } @@ -634,6 +699,14 @@ func (p *crewPicker) rows(width, n int, pal palette, hover int, a *app) []string if p.current == config.CrewCustom { out = append(out, pal.dim(fit(crewCustomLine, width))) } + // AND THE PICK, a fact about the saved crew like the custom line above it. + // It is said only when it is off the default — the table is where the + // seats have always come from, and a line naming it would say something + // nobody chose — and it takes no cursor and no ground, because the pick is + // not staged here: a word typed at the door writes it at once. + if p.pick != "" && p.pick != config.CrewPickTable { + out = append(out, pal.dim(fit(crewPickLead+p.pick, width))) + } // AND THE ROW NOBODY WROTE, said last among the readings and before the door // out: it is the one fact on this list that is about the person's own five // rows rather than about the three on offer, and the answer to it is the @@ -697,9 +770,9 @@ func (a *app) crewListing() string { source := config.CrewSourceAt(a.profileDir) var out strings.Builder // THE WORDS ARE THE SAME IN EITHER FAMILY, so the listing names the pool - // whenever it is not the shipped one: an all listing that looked exactly like - // an open one is the ambiguity this whole feature is about. - if source != config.CrewSourceOpen { + // whenever it is not the default one: two listings that looked exactly alike + // over two different sets of ids is the ambiguity this whole feature is about. + if source != config.DefaultCrewSource { out.WriteString(crewSourceLead + " · " + source + " models\n") } for at, preset := range config.CrewPresets { diff --git a/internal/tui3/crew_test.go b/internal/tui3/crew_test.go index 7112fda523..6c09e54d81 100644 --- a/internal/tui3/crew_test.go +++ b/internal/tui3/crew_test.go @@ -147,7 +147,15 @@ func TestCrewAppliesAPresetAndConfirmsInOneLine(t *testing.T) { if strings.Count(text, "\n") != 0 { t.Fatalf("the confirmation is more than one line:\n%s", text) } - for _, part := range []string{"crew → max", "brain kimi-k3", "hands glm-5.3", "checks kimi-k3"} { + // The three class names come from the table rather than being spelled here: + // a preset that moves onto better models must not leave this test asserting + // the ids it used to name. + for _, part := range []string{ + "crew → max", + "brain " + modelBase(want[config.ModelTierMastermind]), + "hands " + modelBase(want[config.ModelTierWorker]), + "checks " + modelBase(want[config.ModelTierHigh]), + } { if !strings.Contains(text, part) { t.Errorf("the confirmation is missing %q: %q", part, text) } @@ -161,7 +169,10 @@ func TestCrewAppliesAPresetAndConfirmsInOneLine(t *testing.T) { } } -// AN UNKNOWN WORD CHANGES NOTHING AND SHOWS THE THREE. +// AN UNKNOWN WORD CHANGES NOTHING AND SAYS THE SIX — the three presets and the +// three pick words — the shape every choice this surface refuses takes +// (effortchip.go's [app.runEffort]): a refusal that only said "no" would leave +// a person guessing at a word they were one letter away from. func TestCrewRefusesAWordThatIsNotOneOfTheThree(t *testing.T) { a, dir := sheetApp(t) a.slash("/crew cheap") @@ -170,14 +181,20 @@ func TestCrewRefusesAWordThatIsNotOneOfTheThree(t *testing.T) { t.Fatalf("an unknown word changed the crew to %q", got) } text := lastNote(t, a) - if !strings.Contains(text, "not one of the three") { + if !strings.Contains(text, "not a crew word") { t.Fatalf("the refusal reads %q", text) } - for _, preset := range config.CrewPresets { - if !strings.Contains(text, preset) { - t.Errorf("the refusal does not offer %q:\n%s", preset, text) + words := make([]string, 0, len(config.CrewPresets)+len(config.CrewPicks)) + words = append(words, config.CrewPresets...) + words = append(words, config.CrewPicks...) + for _, word := range words { + if !strings.Contains(text, word) { + t.Errorf("the refusal does not offer %q:\n%s", word, text) } } + if got := config.CrewPickAt(dir); got != config.CrewPickTable { + t.Fatalf("an unknown word changed the pick to %q", got) + } } // A CREW SOMEBODY ASSEMBLED THEMSELVES READS AS CUSTOM, and the listing says how @@ -328,32 +345,32 @@ func TestTheCrewChooserNamesTheTwoFamiliesAboveThePresets(t *testing.T) { // THE SHIPPED FAMILY IS THE WORD LIFTED, and it is the only one: the word // the next enter writes is the fact the row answers, and a second lifted // word would be a second answer on a row with one question. - if !strings.Contains(rows[2], a.pal.accent("open models")) { - t.Fatalf("the open family is not the word in force:\n%q", rows[2]) + if !strings.Contains(rows[2], a.pal.accent("all models")) { + t.Fatalf("the all family is not the word in force:\n%q", rows[2]) } - if strings.Contains(rows[2], a.pal.accent("all models")) { + if strings.Contains(rows[2], a.pal.accent("open models")) { t.Fatalf("both families are lifted at once:\n%q", rows[2]) } // AND ←→ MOVES THE WORD while the preset cursor keeps its place and its // mark: the family is a second axis, not a row the ↑↓ list gained. a.crewPickerKey(key("left")) rows = a.overlayRows(a.width, a.overlayHeight()) - if !strings.Contains(rows[2], a.pal.accent("all models")) { - t.Fatalf("← did not walk the family to all:\n%q", rows[2]) + if !strings.Contains(rows[2], a.pal.accent("open models")) { + t.Fatalf("← did not walk the family to open:\n%q", rows[2]) } - // AND THE PRESETS BELOW IT ANSWER IN THAT FAMILY: a header saying all above - // rows still naming open models is the contradiction this chooser exists to - // prevent, and it is the one regression a header-only check would pass. + // AND THE PRESETS BELOW IT ANSWER IN THAT FAMILY: a header saying open above + // rows still naming frontier models is the contradiction this chooser exists + // to prevent, and it is the one regression a header-only check would pass. for i, preset := range config.CrewPresets { - if row := plain(rows[3+2*i]); !strings.Contains(row, config.CrewLineFor(config.CrewSourceAll, preset)) { - t.Errorf("row %d under the all family is not %s's all line:\n%q", 3+2*i, preset, row) + if row := plain(rows[3+2*i]); !strings.Contains(row, config.CrewLineFor(config.CrewSourceOpen, preset)) { + t.Errorf("row %d under the open family is not %s's open line:\n%q", 3+2*i, preset, row) } } // AND THE IN-FORCE MARK DOES NOT MOVE WITH THE FAMILY. The row that wears the // selected ground is the preset the profile holds IN THE FAMILY IT HOLDS, so // once ←→ lifts the family no row on screen is in force: one that kept the // mark would name models the profile does not run. - if strings.Contains(rows[3+2*1], a.pal.accent("balanced — "+config.CrewLineFor(config.CrewSourceAll, config.CrewBalanced))) { + if strings.Contains(rows[3+2*1], a.pal.accent("balanced — "+config.CrewLineFor(config.CrewSourceOpen, config.CrewBalanced))) { t.Errorf("the balanced row is still marked in force after ←:\n%q", rows[3+2*1]) } if a.crewPick.cursor != 1 { @@ -420,12 +437,21 @@ func TestChoosingAFamilyAndAPresetWritesTheSourceRow(t *testing.T) { if a.crewPick.open { t.Fatal("enter left the chooser open") } - if got := config.CrewSourceAt(dir); got != config.CrewSourceAll { - t.Fatalf("enter wrote the family %q, want all", got) + // ← walked the family OFF the default, so the row is written: this is the + // case the family row exists for, and the one a write that skipped an + // unchanged family would still have to land. + if got := config.CrewSourceAt(dir); got != config.CrewSourceOpen { + t.Fatalf("enter wrote the family %q, want open", got) } if got := config.CrewAt(dir); got != config.CrewBalanced { t.Fatalf("enter applied %q, want balanced", got) } + open, _ := config.CrewModelsForSource(config.CrewSourceOpen, config.CrewBalanced) + for _, tier := range config.ModelTiers { + if got := config.TierModelAt(dir, tier); got != open[tier] { + t.Errorf("enter wrote %s as %q, want the open family's %q", tier, got, open[tier]) + } + } // The neighbour survived the one write. data, err := os.ReadFile(config.BudgetConfigPath(dir)) if err != nil { @@ -448,9 +474,9 @@ func TestChoosingAFamilyAndAPresetWritesTheSourceRow(t *testing.T) { // A ROW NOBODY WROTE READS AS THE SHIPPED FAMILY, and so does a word this build // does not know: the chooser offers two options and a stray string off the disk // is not a third. -func TestTheCrewSourceRowDefaultsToOpen(t *testing.T) { +func TestTheCrewSourceRowDefaultsToAll(t *testing.T) { dir := t.TempDir() - if got := config.CrewSourceAt(dir); got != config.CrewSourceOpen { + if got := config.CrewSourceAt(dir); got != config.CrewSourceAll { t.Fatalf("a profile with no row read %q", got) } for _, row := range []string{ @@ -461,15 +487,15 @@ func TestTheCrewSourceRowDefaultsToOpen(t *testing.T) { if err := os.WriteFile(config.BudgetConfigPath(dir), []byte(row), 0o600); err != nil { t.Fatal(err) } - if got := config.CrewSourceAt(dir); got != config.CrewSourceOpen { - t.Fatalf("the row %s read %q, want open", row, got) + if got := config.CrewSourceAt(dir); got != config.CrewSourceAll { + t.Fatalf("the row %s read %q, want all", row, got) } } - if err := os.WriteFile(config.BudgetConfigPath(dir), []byte(`{"models.crew.source": "all"}`), 0o600); err != nil { + if err := os.WriteFile(config.BudgetConfigPath(dir), []byte(`{"models.crew.source": "open"}`), 0o600); err != nil { t.Fatal(err) } - if got := config.CrewSourceAt(dir); got != config.CrewSourceAll { - t.Fatalf("the all row read %q", got) + if got := config.CrewSourceAt(dir); got != config.CrewSourceOpen { + t.Fatalf("the open row read %q", got) } } @@ -500,9 +526,14 @@ func TestStatusNamesTheCrewUnderTheModel(t *testing.T) { t.Fatalf("/status says nothing about the crew:\n%s", text) } // The preset word first, then the same three class names the confirmation - // prints — base names, in [config.CrewClasses]'s own order. + // prints — base names, in [config.CrewClasses]'s own order, read off the + // table rather than spelled here. + max, _ := config.CrewModels(config.CrewMax) for _, want := range []string{ - config.CrewMax, "brain kimi-k3", "hands glm-5.3", "checks kimi-k3", + config.CrewMax, + "brain " + modelBase(max[config.ModelTierMastermind]), + "hands " + modelBase(max[config.ModelTierWorker]), + "checks " + modelBase(max[config.ModelTierHigh]), } { if !strings.Contains(lines[at], want) { t.Errorf("the crew line lost %q: %q", want, lines[at]) @@ -861,3 +892,103 @@ func TestTheStatusSheetCarriesTheCrewUnderTheModel(t *testing.T) { } t.Fatalf("the sheet has no crew line: %+v", items) } + +// ── the pick row ──────────────────────────────────────────────────────────── + +// THE THREE PICK WORDS ARE DOOR WORDS, beside the presets: `/crew learn` says +// where the seats come from and moves no model id, and the confirmation, the +// crew word and the segment all carry it. `/crew table` is the way back. +func TestCrewTakesThePickWords(t *testing.T) { + a, dir := sheetApp(t) + a.slash("/crew learn") + + if got := config.CrewPickAt(dir); got != config.CrewPickLearn { + t.Fatalf("/crew learn left the pick reading %q", got) + } + if got := config.CrewAt(dir); got != config.DefaultCrew { + t.Fatalf("a pick word moved the crew to %q", got) + } + // THE CREW WORD AND THE SEGMENT SAY IT: `balanced · learn` on the page, + // `crew balanced · learn` on the status line — the word a person reads in + // /status is the word the frame carries. + if word := a.crewWord(); !strings.Contains(word, "· learn") { + t.Fatalf("the crew word reads %q, want the pick beside the preset", word) + } + if seg := a.crewSegment(); seg != "crew balanced · learn" { + t.Fatalf("the segment reads %q, want crew balanced · learn", seg) + } + if text := lastNote(t, a); !strings.Contains(text, "crew → "+config.DefaultCrew+" · learn") { + t.Fatalf("the confirmation reads %q, want the pick named", text) + } + + a.slash("/crew table") + if got := config.CrewPickAt(dir); got != config.CrewPickTable { + t.Fatalf("/crew table left the pick reading %q", got) + } + if seg := a.crewSegment(); seg != "crew "+config.DefaultCrew { + t.Fatalf("the segment reads %q after the way back, want the plain crew word", seg) + } +} + +// THE CHOOSER NAMES THE PICK when it is off the table, and says nothing about +// the default: the table is where the seats have always come from, and a line +// naming it would say something nobody chose. +func TestTheCrewChooserNamesThePickWhenItIsOffTheTable(t *testing.T) { + a, _ := sheetApp(t) + a.slash("/crew learn") + a.slash("/crew") + text := plain(strings.Join(a.overlayRows(a.width, a.overlayHeight()), "\n")) + if !strings.Contains(text, "picked from learn") { + t.Fatalf("the chooser does not name the pick:\n%s", text) + } + a.crewPick.close() + a.slash("/crew table") + a.slash("/crew") + text = plain(strings.Join(a.overlayRows(a.width, a.overlayHeight()), "\n")) + if strings.Contains(text, "picked from") { + t.Fatalf("the chooser names the default pick:\n%s", text) + } +} + +// THE PICK ROW SITS UNDER THE CREW ROW AND CYCLES, like every other enum row +// on the sheet: the crew word above it says how much to spend, and this says +// where the models for that money come from. +func TestThePickRowSitsUnderTheCrewRowAndCycles(t *testing.T) { + a, dir := sheetApp(t) + a.openSettings() + toProviders(t, a) + + // THE PLACE: directly under the crew row, before the classes it seats. A + // pick row a tab away from the crew word would be two errands on two + // screens for one decision. + crew, pick := -1, -1 + for i, item := range a.sheet.items { + switch item.row.Key { + case config.KeyCrew: + crew = i + case config.KeyCrewPick: + pick = i + } + } + if crew < 0 || pick < 0 { + t.Fatalf("the Providers tab holds no crew/pick pair (crew %d, pick %d)", crew, pick) + } + if pick != crew+1 { + t.Fatalf("the pick row sits at %d with the crew at %d, want it directly under", pick, crew) + + } + + cursorTo(t, a, config.KeyCrewPick) + drive(t, a, key("enter")) + if got := config.CrewPickAt(dir); got != config.CrewPickCatalog { + t.Fatalf("enter on table landed on %q, want %q", got, config.CrewPickCatalog) + } + drive(t, a, key("enter")) + if got := config.CrewPickAt(dir); got != config.CrewPickLearn { + t.Fatalf("enter on catalog landed on %q, want %q", got, config.CrewPickLearn) + } + drive(t, a, key("enter")) + if got := config.CrewPickAt(dir); got != config.CrewPickTable { + t.Fatalf("enter on learn landed on %q, want %q", got, config.CrewPickTable) + } +} diff --git a/internal/tui3/onboarding.go b/internal/tui3/onboarding.go index 20f8df326c..6881df2030 100644 --- a/internal/tui3/onboarding.go +++ b/internal/tui3/onboarding.go @@ -1468,8 +1468,8 @@ const setupNoMatchWord = "nothing matches · backspace widens it" // is deliberately NOT [config.CrewLine]. // // The registry's line is written for somebody who already has the vocabulary: -// `deepseek works, glm-flash thinks · pennies a day` names four models and makes -// a claim about money. Both halves are wrong here. Nobody meeting codeaf for the +// `glm-flash works, checks and thinks · pennies a day` names one model three +// times and makes a claim about money. Both halves are wrong here. Nobody meeting codeaf for the // first time can rank `glm-flash` against `kimi-k3`, and — the harder rule — // THIS SCREEN MAKES NO CLAIM ABOUT WHAT ANYTHING COSTS. "Pennies a day" is a // forecast about somebody else's usage on somebody else's pricing, said on the @@ -1495,7 +1495,7 @@ func crewChoiceWord(preset string) string { return word } // The fallback is for a preset this screen has not been taught, and it is the - // open family's line: setupCrewRows runs inside the draw, so reading the profile + // default family's line: setupCrewRows runs inside the draw, so reading the profile // here would put a profile read on the frame clock, and the seats this screen // shows already take the family from a value read at the door. return config.CrewLine(preset) diff --git a/internal/tui3/settings.go b/internal/tui3/settings.go index d8345af435..87109a9eab 100644 --- a/internal/tui3/settings.go +++ b/internal/tui3/settings.go @@ -378,7 +378,22 @@ var settingUI = map[string]settingMeta{ config.KeyCrewSource: { tab: tabProviders, label: "model family", widget: widgetCycle, about: "which models the crew word draws from: open weights, or the whole " + - "catalog with closed and frontier models in it. Open is the default.", + "catalog with closed and frontier models in it. All is the default.", + }, + // WHERE THE CREW'S MODELS COME FROM, beside the crew word itself. The crew + // row says how much to spend and this says where the models for that money + // are read from when a class row does not hold a person's own id: the rows + // this build measured, or the same budgets recomputed off the catalog on + // every read, with or without what the Model Pool and the person's own + // judged runs measured. The about is the registry's own hint whole, not its + // first sentence, because the three words are the answer and the first + // sentence alone would send a person hunting for what catalog means. + config.KeyCrewPick: { + tab: tabProviders, label: "picked from", widget: widgetCycle, + about: "where the crew's models come from. table: the rows we measured. " + + "catalog: recomputed from today's published prices and scores at your " + + "crew's budget. learn: catalog plus the Model Pool's measurements and " + + "your own judged runs.", }, config.KeyTierLowModel: { tab: tabProviders, label: "small work", widget: widgetSelect, @@ -657,6 +672,24 @@ var settingUI = map[string]settingMeta{ about: "Rich icons normally; plain symbols when your terminal needs them.", widget: widgetCycle, }, + // The pool row sits with the models it chooses among: the word decides + // whether this machine reads the shared measurements and whether its own + // runs are sent back, and nothing about the code leaves either way. + config.KeyModelPool: { + tab: tabProviders, label: "model pool", widget: widgetCycle, + about: "codeaf picks your models from the public Model Pool, and your runs " + + "improve it. On by default: what leaves is computed, text-free numbers " + + "under a per-install nonce, never code, prompts or paths. read uses " + + "the pool and sends nothing; off does neither.", + }, + // The key sits under the pool row it guards: a private relay is the same + // code under another keypair, and this is where the install is told whose + // signature to trust. Blank is the key built into this binary. + config.KeyModelPoolPublicKey: { + tab: tabProviders, label: "pool key", widget: widgetText, + about: "the public key a Model Pool index must be signed with. Blank " + + "trusts the key built into codeaf; set it to read a relay of your own.", + }, config.KeyVisionModel: { tab: tabProviders, label: "looking", widget: widgetSelect, about: "the model that looks at images. Blank picks one that can see.", @@ -743,7 +776,7 @@ func init() { // same row, the same write, the same live seam onto [app.switchModel] — only // the word above the Models section changed. crew := settingUI[config.KeyCrew] - crew.about = crewAbout(config.CrewSourceOpen) + crew.about = crewAbout(config.DefaultCrewSource) settingUI[config.KeyCrew] = crew talk := settingUI[config.ModelSettingKey(talkSlot)] @@ -757,8 +790,8 @@ func init() { // with its own sentence, then what makes the row read custom. The sentences are // [config.CrewLineFor]'s IN THE FAMILY ON SCREEN, so the panel and /crew say the // same words about the same thing and neither names an open model above frontier -// ids. The init-time value is the open family; [sheet.metaFor] re-says it from the -// profile, which is where a family that is not the default comes from. +// ids. The init-time value is the default family; [sheet.metaFor] re-says it from +// the profile, which is where a family that is not the default comes from. func crewAbout(source string) string { said := make([]string, 0, len(config.CrewPresets)) for _, preset := range config.CrewPresets { @@ -805,6 +838,10 @@ func modelsSectionOrder() []string { config.KeyRouting, config.KeyPromptProfile, config.KeyCrew, + // THE PICK ANSWERS THE CREW WORD'S OWN QUESTION ONE LEVEL DOWN — where + // the models for that budget come from — so it reads directly under the + // crew word, before the classes it seats. + config.KeyCrewPick, } for _, tier := range roles.Tiers { order = append(order, tierSettingKey(tier)) diff --git a/relay/README.md b/relay/README.md new file mode 100644 index 0000000000..4e09062baa --- /dev/null +++ b/relay/README.md @@ -0,0 +1,39 @@ +# Model Pool relay + +This Worker stores one running total per install, cell and day — the per-install +sheets it folds submitted rows into — plus a per-install per-day quota counter, +and the last published index document and its signature. + +It never stores a client address and keeps no request log, and it reads the +32-byte private signing seed only from the `POOL_SIGNING_KEY` secret, which +lives in Cloudflare, not in this directory. + +It serves, all under `/pool/`: `POST /pool/v1/rows` for submissions, +`GET /pool/index.json` for the published index, `GET /pool/index.json.sig` for +its signature, and `GET /pool/healthz`; every other path answers 404 with an +empty body. + +## Priming + +The first published index was primed once, from the same measured runs the +binary's embedded seed encodes, so a machine that has never fetched a document +and one that has read the first published one carry the same numbers. + +`tools/prime.py` is that one-off. Run by the operator over the private run +ledgers, which never enter this repository and stay outside it, it rebuilds the +ordinary rows those runs imply — one row per seat a run scored, on the same +wire, with the same seat and score mapping the seed was folded from — and +submits them to `POST /v1/rows`. Its `--dry-run` prints the rows and checks them +against the seed without sending anything. + +It gives every originating session one install id — the first 32 hex characters +of the hash of the session name — so a session's runs fold under one install the +same way they would have had that session submitted them itself. Every row it +sends carries the judge id `codeaf/reviewer`: these runs were scored by review +rather than by a judge model, and one judge id keeps the relay's judge-severity +fit neutral. + +Nothing is counted twice. A client holds one index document at a time, and the +reader takes whichever of the fetched document and the embedded seed is newer by +its generated date rather than summing them, so a row the relay counts once is +already the seed's cell and not a second observation on top of it. diff --git a/relay/package.json b/relay/package.json new file mode 100644 index 0000000000..500852336b --- /dev/null +++ b/relay/package.json @@ -0,0 +1,8 @@ +{ + "name": "codeaf-pool-relay", + "private": true, + "type": "module", + "scripts": { + "test": "node --test test" + } +} diff --git a/relay/src/index.js b/relay/src/index.js new file mode 100644 index 0000000000..fabbee0550 --- /dev/null +++ b/relay/src/index.js @@ -0,0 +1,85 @@ +// index.js builds the signed index document and signs it. +// +// The document is what internal/pool/index.Parse reads (see +// internal/pool/index/seed.json for a complete example). Keys sit in a fixed +// order and the text is JSON with no spaces, so the same cells always render +// the same bytes. +// +// signIndex signs under Ed25519 through WebCrypto, which is the same API +// under Workers and under node ≥ 20 through globalThis.crypto.subtle. + +const SCHEMA = 1; + +// buildIndex renders the document for cells. The keys are written in the fixed +// order the reader expects; JSON.stringify keeps the order of object keys. +export function buildIndex(cells, { version, generated, minInstalls, judges }) { + const doc = { + schema: SCHEMA, + version, + generated, + min_installs: minInstalls, + judges, + rubrics: { role_quality: 1 }, + aliases: {}, + metrics: { + role_quality: { kind: 'gaussian', unit: 'score', dims: ['role', 'model'] }, + }, + cells: cells.map((c) => ({ + metric: c.metric, + role: c.role, + model: c.model, + mean: c.mean, + sd: c.sd, + n: c.n, + installs: c.installs, + })), + }; + return JSON.stringify(doc); +} + +// signIndex signs docBytes and answers the detached signature as standard +// base64. seedB64 is the 32-byte private seed and publicB64 the 32-byte public +// key, both standard base64; together they import as one Ed25519 JWK. +export async function signIndex(docBytes, seedB64, publicB64) { + const seed = fromBase64(seedB64); + const pub = fromBase64(publicB64); + const key = await crypto.subtle.importKey( + 'jwk', + { + kty: 'OKP', + crv: 'Ed25519', + d: toBase64Url(seed), + x: toBase64Url(pub), + }, + { name: 'Ed25519' }, + false, + ['sign'], + ); + const bytes = typeof docBytes === 'string' ? new TextEncoder().encode(docBytes) : docBytes; + const signature = await crypto.subtle.sign({ name: 'Ed25519' }, key, bytes); + return toBase64(new Uint8Array(signature)); +} + +// fromBase64 decodes standard base64 text (whitespace around it ignored). +function fromBase64(text) { + const raw = atob(String(text).trim()); + const out = new Uint8Array(raw.length); + for (let i = 0; i < raw.length; i++) { + out[i] = raw.charCodeAt(i); + } + return out; +} + +// toBase64 renders bytes as standard base64. +function toBase64(bytes) { + let s = ''; + for (const b of bytes) { + s += String.fromCharCode(b); + } + return btoa(s); +} + +// toBase64Url renders bytes as base64url, which a JWK field wants. +function toBase64Url(bytes) { + return toBase64(bytes).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} diff --git a/relay/src/schema.js b/relay/src/schema.js new file mode 100644 index 0000000000..c32b46423e --- /dev/null +++ b/relay/src/schema.js @@ -0,0 +1,138 @@ +// schema.js validates the submit wire. The Go client writes exactly these +// rows (internal/pool/outbox.Row carrying internal/pool/record.Row), so this +// module refuses anything the client would not have sent. +// +// One NDJSON line is an outer envelope plus the payload it carries: +// {"schema":1,"day":"YYYY-MM-DD","nonce":"<32 lowercase hex>","payload":{...}} +// The payload is the record.Row fields: +// {"schema":1,"metric":"role_quality","role":...,"model":...,"score":..., +// "judge":...,"door":...,"size":...,"day":"YYYY-MM-DD"} +// +// Everything here is pure: no KV, no fetch, no clock of its own. + +// The one metric the client records. +const METRIC = 'role_quality'; + +const ROLES = new Set(['worker', 'high', 'mastermind']); +const DOORS = new Set(['task', 'do']); +const SIZES = new Set(['S', 'M', 'L']); + +// A model id is "/". The Go side draws both halves from a closed +// set of provider and model names; this is the loose shape it always matches. +const MODEL_RE = /^[a-z0-9._-]+\/[a-z0-9._:-]+$/i; +const DAY_RE = /^\d{4}-\d{2}-\d{2}$/; +const HEX32_RE = /^[0-9a-f]{32}$/; + +// A row's day may be at most one day ahead of now (UTC): a clock a little off +// is fine, a date in the future is not. +const ONE_DAY_MS = 86_400_000; + +// utcDay reads a "YYYY-MM-DD" day as milliseconds since the epoch, UTC. +function utcDay(day) { + const y = Number(day.slice(0, 4)); + const m = Number(day.slice(5, 7)); + const d = Number(day.slice(8, 10)); + return Date.UTC(y, m - 1, d); +} + +// dayError answers why a day string is unusable, or null when it is fine. +function dayError(day, now) { + if (typeof day !== 'string' || !DAY_RE.test(day)) { + return 'day must be YYYY-MM-DD'; + } + const todayMs = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); + if (utcDay(day) - todayMs > ONE_DAY_MS) { + return 'day is more than one day in the future'; + } + return null; +} + +// validateRow reads one NDJSON line. It answers {row, error}: row is the +// parsed envelope with its payload when the line is good, and null with a +// message otherwise. `now` bounds how far ahead the day may be. +export function validateRow(line, now = new Date()) { + if (typeof line !== 'string' || line.trim() === '') { + return fail('line is empty'); + } + let outer; + try { + outer = JSON.parse(line); + } catch { + return fail('line is not json'); + } + if (outer === null || typeof outer !== 'object' || Array.isArray(outer)) { + return fail('row is not an object'); + } + if (outer.schema !== 1) { + return fail('schema must be 1'); + } + const dayBad = dayError(outer.day, now); + if (dayBad !== null) { + return fail(dayBad); + } + if (typeof outer.nonce !== 'string' || !HEX32_RE.test(outer.nonce)) { + return fail('nonce must be 32 lowercase hex'); + } + const payload = outer.payload; + if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) { + return fail('payload is not an object'); + } + if (payload.schema !== 1) { + return fail('payload schema must be 1'); + } + if (payload.metric !== METRIC) { + return fail(`metric must be ${METRIC}`); + } + if (!ROLES.has(payload.role)) { + return fail('role must be worker, high or mastermind'); + } + if (typeof payload.model !== 'string' || !MODEL_RE.test(payload.model)) { + return fail('model must be /'); + } + if (typeof payload.score !== 'number' || !Number.isFinite(payload.score) + || payload.score < 0 || payload.score > 100) { + return fail('score must be a number in 0-100'); + } + if (typeof payload.judge !== 'string' || !MODEL_RE.test(payload.judge)) { + return fail('judge must be /'); + } + if (!DOORS.has(payload.door)) { + return fail('door must be task or do'); + } + if (!SIZES.has(payload.size)) { + return fail('size must be S, M or L'); + } + const payloadDayBad = dayError(payload.day, now); + if (payloadDayBad !== null) { + return fail(payloadDayBad); + } + return { + row: { + schema: 1, + day: outer.day, + nonce: outer.nonce, + payload: { + schema: 1, + metric: payload.metric, + role: payload.role, + model: payload.model, + score: payload.score, + judge: payload.judge, + door: payload.door, + size: payload.size, + day: payload.day, + }, + }, + error: null, + }; +} + +// validateInstall answers whether a request's X-Codeaf-Install header is an +// install id: 32 lowercase hex characters, the install's own nonce. +export function validateInstall(header) { + return typeof header === 'string' && HEX32_RE.test(header); +} + +function fail(message) { + return { row: null, error: message }; +} diff --git a/relay/src/sheet.js b/relay/src/sheet.js new file mode 100644 index 0000000000..14474a049d --- /dev/null +++ b/relay/src/sheet.js @@ -0,0 +1,248 @@ +// sheet.js folds a install's rows into running totals and aggregates all +// installs' totals into the cells of an index document. +// +// Each stored triple is {n, s, s2}: how many scores, their sum and their sum +// of squares. A triple is a running total, so two copies of the same triple +// carry the same or more observations and the larger one is the more complete +// — that is what makes `join` a least upper bound and the fold a CRDT. +// +// Everything here is pure: plain values in, plain values out. No KV, no fetch. + +// METRIC_ROLE_QUALITY is the one metric the sheets carry. +const METRIC_ROLE_QUALITY = 'role_quality'; + +// ZERO is an empty triple. +const ZERO = Object.freeze({ n: 0, s: 0, s2: 0 }); + +// cellKey renders the place one triple lives: the five dimensions that are +// kept apart, in a fixed order. The day is the KV key's own segment and is not +// part of this key. The dimensions hold no "|" (the model pattern forbids it), +// so the key splits back cleanly. +export function cellKey(payload) { + return `${payload.role}|${payload.model}|${payload.judge}|${payload.door}|${payload.size}`; +} + +// fold adds one score to a triple and answers the new triple. A missing +// triple folds as if it were empty. +export function fold(triple, score) { + const t = triple || ZERO; + return { n: t.n + 1, s: t.s + score, s2: t.s2 + score * score }; +} + +// join is the componentwise maximum of two triples — the least upper bound of +// two running totals of the same measurement. It is commutative, associative +// and idempotent. +export function join(a, b) { + const x = a || ZERO; + const y = b || ZERO; + return { + n: Math.max(x.n, y.n), + s: Math.max(x.s, y.s), + s2: Math.max(x.s2, y.s2), + }; +} + +// splitKey reads a cellKey back into its five dimensions. +function splitKey(key) { + const [role, model, judge, door, size] = key.split('|'); + return { role, model, judge, door, size }; +} + +// judgeSeverity fits one additive judge severity per judge over the finest +// cells (one per cellKey), so a judge who scores high or low on everything is +// taken out of every score. It alternates beta_j = weighted mean over cells +// scored by j of (cellMean - mu_rm) with mu_rm = weighted mean over judges of +// (cellMean - beta_j), weights n, five sweeps, re-centring beta to weighted +// mean zero each sweep. It answers {beta, judges}. +function judgeSeverity(cells) { + const judges = [...new Set(cells.map((c) => c.judge))].sort(); + const beta = new Map(judges.map((j) => [j, 0])); + const mu = new Map(); + for (const c of cells) { + mu.set(rmOf(c), 0); + } + for (let sweep = 0; sweep < 5; sweep++) { + // beta_j from the current mu_rm. + const num = new Map(); + const den = new Map(); + for (const c of cells) { + const w = c.n; + num.set(c.judge, (num.get(c.judge) || 0) + w * (c.mean - (mu.get(rmOf(c)) || 0))); + den.set(c.judge, (den.get(c.judge) || 0) + w); + } + for (const j of judges) { + beta.set(j, den.get(j) ? num.get(j) / den.get(j) : 0); + } + // Re-centre beta so its weighted mean is zero; the split between beta and + // mu is then pinned. + let bnum = 0; + let bden = 0; + for (const c of cells) { + bnum += c.n * beta.get(c.judge); + bden += c.n; + } + const centre = bden ? bnum / bden : 0; + for (const j of judges) { + beta.set(j, beta.get(j) - centre); + } + // mu_rm from the fresh beta_j. + const mnum = new Map(); + const mden = new Map(); + for (const c of cells) { + const w = c.n; + const rm = rmOf(c); + mnum.set(rm, (mnum.get(rm) || 0) + w * (c.mean - beta.get(c.judge))); + mden.set(rm, (mden.get(rm) || 0) + w); + } + for (const rm of mu.keys()) { + mu.set(rm, mden.get(rm) ? mnum.get(rm) / mden.get(rm) : 0); + } + } + return { beta, judges }; +} + +function rmOf(cell) { + return `${cell.role}|${cell.model}`; +} + +// median of a numeric list (the list is copied, not sorted in place). +function median(values) { + const xs = [...values].sort((a, b) => a - b); + const mid = Math.floor(xs.length / 2); + return xs.length % 2 ? xs[mid] : (xs[mid - 1] + xs[mid]) / 2; +} + +// huberMean is the Huber M-estimate of a location from values: scale from the +// median absolute deviation (1.4826 · MAD), tuning 1.345, twenty iterations. +// Each value's weight is 1 inside c·scale and c·scale/|x-mu| beyond it, so a +// single far-off value cannot drag the estimate. +function huberMean(values) { + const centre = median(values); + const mad = median(values.map((v) => Math.abs(v - centre))); + const scale = 1.4826 * mad; + if (!(scale > 0)) { + return values.reduce((a, b) => a + b, 0) / values.length; + } + const c = 1.345 * scale; + let mu = centre; + for (let i = 0; i < 20; i++) { + let num = 0; + let den = 0; + for (const v of values) { + const dev = Math.abs(v - mu); + const w = dev <= c ? 1 : c / dev; + num += w * v; + den += w; + } + if (den === 0) { + break; + } + mu = num / den; + } + return mu; +} + +// aggregate turns sheet entries into the cells of an index document. +// +// entries are {install, day, key, triple}: one install's running total for one +// cellKey on one day. Cells group by (role, model); each install's mean over +// its triples is taken after judge severity is removed, and the cell mean is +// the Huber M-estimate over those install means once there are at least five +// installs, the n-weighted mean of them otherwise. A cell whose installs are +// below minInstalls is still published, but sorted after the cells that meet +// it. +export function aggregate(entries, { minInstalls = 0 } = {}) { + const cells = []; + for (const entry of entries) { + const dims = splitKey(entry.key); + const n = entry.triple.n; + if (!(n > 0)) { + continue; + } + cells.push({ ...dims, n, mean: entry.triple.s / n }); + } + const { beta } = judgeSeverity(cells); + + // Group the adjusted entries by role and model, and keep each install's own + // observations so its mean over them can be taken. + const groups = new Map(); + for (const entry of entries) { + const n = entry.triple.n; + if (!(n > 0)) { + continue; + } + const dims = splitKey(entry.key); + const rm = `${dims.role}|${dims.model}`; + let group = groups.get(rm); + if (!group) { + group = { role: dims.role, model: dims.model, installs: new Map(), n: 0, s: 0, s2: 0 }; + groups.set(rm, group); + } + // Judge severity removal shifts each score by -beta[judge]; over a + // triple of n scores and sum s that is a shift of n·beta in the sum. + const shift = beta.get(dims.judge) || 0; + const adjustedSum = entry.triple.s - n * shift; + group.n += n; + group.s += entry.triple.s; + group.s2 += entry.triple.s2; + const install = group.installs.get(entry.install); + if (install) { + install.n += n; + install.s += adjustedSum; + } else { + group.installs.set(entry.install, { n, s: adjustedSum }); + } + } + + const out = []; + for (const group of groups.values()) { + const installMeans = []; + let weightSum = 0; + let weightedSum = 0; + for (const install of group.installs.values()) { + const mean = install.s / install.n; + installMeans.push(mean); + weightSum += install.n; + weightedSum += install.n * mean; + } + const installs = installMeans.length; + const mean = installs >= 5 + ? huberMean(installMeans) + : (weightSum ? weightedSum / weightSum : 0); + // Pooled within-cell standard deviation from the raw triples. + const sd = group.n >= 2 + ? Math.sqrt(Math.max(0, (group.s2 - (group.s * group.s) / group.n) / (group.n - 1))) + : 0; + out.push({ + metric: METRIC_ROLE_QUALITY, + role: group.role, + model: group.model, + mean, + sd, + n: group.n, + installs, + }); + } + + // Cells that meet minInstalls sort first; below it they follow. Ties break on + // mean then role then model, so the document is a property of its entries. + out.sort((a, b) => { + const am = a.installs >= minInstalls ? 0 : 1; + const bm = b.installs >= minInstalls ? 0 : 1; + if (am !== bm) return am - bm; + if (a.mean !== b.mean) return b.mean - a.mean; + if (a.role !== b.role) return a.role < b.role ? -1 : 1; + if (a.model !== b.model) return a.model < b.model ? -1 : 1; + return 0; + }); + return out; +} + +// judgesOf answers the sorted judge ids the entries name. +export function judgesOf(entries) { + const judges = new Set(); + for (const entry of entries) { + judges.add(entry.key.split('|')[2]); + } + return [...judges].sort(); +} diff --git a/relay/src/worker.js b/relay/src/worker.js new file mode 100644 index 0000000000..3fa59b2386 --- /dev/null +++ b/relay/src/worker.js @@ -0,0 +1,297 @@ +// worker.js is the Model Pool relay: it accepts measurement rows from +// installs, folds each install's rows into per-install running totals, and +// once an hour publishes a signed, versioned index document and serves it. +// +// The wire is fixed by the Go client (internal/pool/outbox for submit, +// internal/pool/pull and internal/pool/index for the document), so this file +// matches it exactly and adds nothing of its own to it. +// +// It owns only its /pool/ prefix; every other path answers 404 with an empty +// body. It never logs a request and never reads or stores the client address. + +import { validateInstall, validateRow } from './schema.js'; +import { aggregate, cellKey, fold, judgesOf } from './sheet.js'; +import { buildIndex, signIndex } from './index.js'; + +const PREFIX = '/pool'; + +// The submit limits the client already works within. +const MAX_BODY = 256 << 10; // 256 KiB +const MAX_LINES = 200; + +// The KV keys the published document lives under. +const DOC_KEY = 'index/doc'; +const SIG_KEY = 'index/sig'; +const VERSION_KEY = 'index/version'; + +// The lock one reader holds while it republishes; it expires by itself so a +// reader that dies mid-publish does not wedge the rest. +const LOCK_KEY = 'index/publishing'; + +export default { + // fetch serves the four routes under /pool/. The ctx carries waitUntil + // when the platform passes one; reads never require it. + async fetch(request, env, ctx) { + const url = new URL(request.url); + const route = routeOf(url.pathname); + switch (route) { + case '/v1/rows': + if (request.method === 'POST') { + return handleRows(request, env); + } + return notFound(); + case '/index.json': + await refreshInBackground(env, ctx); + return serveStored(request, env, DOC_KEY, 'application/json'); + case '/index.json.sig': + await refreshInBackground(env, ctx); + return serveStored(request, env, SIG_KEY, 'text/plain'); + case '/healthz': + if (request.method === 'GET' || request.method === 'HEAD') { + return new Response(request.method === 'HEAD' ? null : 'ok', { + status: 200, + headers: { 'Content-Type': 'text/plain' }, + }); + } + return notFound(); + default: + return notFound(); + } + }, + + // scheduled folds every install's sheets into one document, signs it, and + // replaces the published copy. A publication with no cells still publishes: + // an empty index is a valid index. + async scheduled(event, env) { + await publish(env); + }, +}; + +// publish folds every install's sheets into one document, signs it, and +// replaces the published copy. The document's version is its publish time +// in unix seconds, which is what reads compare against PUBLISH_EVERY. +async function publish(env) { + const minInstalls = intVar(env, 'MIN_INSTALLS', 3); + const entries = await loadEntries(env); + const cells = aggregate(entries, { minInstalls }); + const judges = judgesOf(entries); + const version = Math.floor(Date.now() / 1000); // monotone by construction + const generated = new Date().toISOString().slice(0, 10); // today, UTC + const doc = buildIndex(cells, { version, generated, minInstalls, judges }); + const sig = await signIndex(doc, env.POOL_SIGNING_KEY, env.POOL_PUBLIC_KEY); + await env.POOL.put(DOC_KEY, doc); + await env.POOL.put(SIG_KEY, sig); + await env.POOL.put(VERSION_KEY, String(version)); +} + +// routeOf answers the path under /pool/, or null when the request is not ours. +function routeOf(pathname) { + if (pathname === PREFIX) { + return '/'; + } + if (pathname.startsWith(PREFIX + '/')) { + return pathname.slice(PREFIX.length); + } + return null; +} + +// notFound is the empty 404 every path outside a served route gets. +function notFound() { + return new Response(null, { status: 404 }); +} + +// json answers one JSON object with a status. +function json(body, status) { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +// handleRows accepts one NDJSON batch from one install. +async function handleRows(request, env) { + const install = request.headers.get('X-Codeaf-Install'); + if (!validateInstall(install)) { + return json({ error: 'missing or bad X-Codeaf-Install' }, 400); + } + const body = await request.text(); + if (new TextEncoder().encode(body).length > MAX_BODY) { + return json({ error: 'body is larger than 256 KiB' }, 413); + } + const lines = []; + for (const [index, raw] of body.split('\n').entries()) { + const line = raw.endsWith('\r') ? raw.slice(0, -1) : raw; + if (line.trim() !== '') { + lines.push({ line, number: index + 1 }); + } + } + if (lines.length > MAX_LINES) { + return json({ error: 'more than 200 lines' }, 413); + } + if (lines.length === 0) { + return json({ error: 'no rows' }, 400); + } + const now = new Date(); + const rows = []; + for (const { line, number } of lines) { + const { row, error } = validateRow(line, now); + if (error !== null) { + return json({ error: `line ${number}: ${error}` }, 400); + } + rows.push(row); + } + + // The per-install per-day quota is checked for the whole batch before + // anything is stored: a batch that would exceed it stores nothing. + const limit = intVar(env, 'ROWS_PER_INSTALL_PER_DAY', 500); + const byDay = new Map(); + for (const row of rows) { + const list = byDay.get(row.day) || []; + list.push(row); + byDay.set(row.day, list); + } + const used = new Map(); + for (const [day, list] of byDay) { + const current = parseInt(await env.POOL.get(quotaKey(install, day)), 10) || 0; + if (current + list.length > limit) { + return json({ error: 'daily quota exceeded' }, 429); + } + used.set(day, current); + } + + // Fold each score into its install's running total for the cell and day. + for (const row of rows) { + const key = `sheet/${install}/${row.day}/${cellKey(row.payload)}`; + const previous = await env.POOL.get(key); + let triple = null; + if (previous !== null) { + try { + triple = JSON.parse(previous); + } catch { + triple = null; + } + } + await env.POOL.put(key, JSON.stringify(fold(triple, row.payload.score))); + } + for (const [day, list] of byDay) { + await env.POOL.put(quotaKey(install, day), String(used.get(day) + list.length)); + } + return json({ accepted: rows.length }, 202); +} + +// quotaKey names one install's per-day quota counter. +function quotaKey(install, day) { + return `quota/${install}/${day}`; +} + +// refreshInBackground republishes the document behind a read when the +// stored version is missing or older than PUBLISH_EVERY seconds. The read +// itself still answers what is stored, so callers never wait on a publish. +async function refreshInBackground(env, ctx) { + if (!ctx || typeof ctx.waitUntil !== 'function') { + return; + } + const every = intVar(env, 'PUBLISH_EVERY', 3600); + const version = await env.POOL.get(VERSION_KEY); + if (!isStale(version, every)) { + return; + } + ctx.waitUntil(guardedPublish(env)); +} + +// isStale answers whether a stored version needs republishing. The version +// is the publish time in unix seconds, so staleness is its age in seconds; +// a missing or unreadable version is always stale. +function isStale(version, every) { + const published = parseInt(version, 10); + if (!Number.isFinite(published)) { + return true; + } + return Math.floor(Date.now() / 1000) - published > every; +} + +// guardedPublish publishes once per stale spell: a reader that finds the +// lock takes it and publishes, and the rest serve what is stored. The +// in-flight promise serialises readers on this isolate, where the check and +// the write would otherwise interleave; the KV lock covers the rest. +let inflight = null; + +function guardedPublish(env) { + if (inflight !== null) { + return inflight; + } + inflight = (async () => { + try { + const held = await env.POOL.get(LOCK_KEY); + if (held !== null) { + return; + } + await env.POOL.put(LOCK_KEY, '1', { expirationTtl: 60 }); + await publish(env); + } finally { + inflight = null; + } + })(); + return inflight; +} + +// serveStored answers one of the published KV values with its caching headers. +async function serveStored(request, env, key, contentType) { + if (request.method !== 'GET' && request.method !== 'HEAD') { + return notFound(); + } + const value = await env.POOL.get(key); + if (value === null) { + return notFound(); + } + const version = await env.POOL.get(VERSION_KEY); + const etag = `"${version === null ? '' : version}"`; + const headers = { + 'Content-Type': contentType, + 'Cache-Control': 'public, max-age=300', + ETag: etag, + }; + const ifNoneMatch = request.headers.get('If-None-Match'); + if (ifNoneMatch !== null && ifNoneMatch.trim() === etag) { + return new Response(null, { status: 304, headers }); + } + return new Response(request.method === 'HEAD' ? null : value, { status: 200, headers }); +} + +// loadEntries reads every sheet triple the store holds, one entry per install, +// cell and day. +async function loadEntries(env) { + const entries = []; + let cursor; + do { + const page = await env.POOL.list({ prefix: 'sheet/', cursor }); + for (const key of page.keys) { + const parts = key.name.slice('sheet/'.length).split('/'); + const install = parts[0]; + const day = parts[1]; + const cell = parts.slice(2).join('/'); + const value = await env.POOL.get(key.name); + if (value === null) { + continue; + } + let triple; + try { + triple = JSON.parse(value); + } catch { + continue; + } + if (!triple || typeof triple.n !== 'number') { + continue; + } + entries.push({ install, day, key: cell, triple }); + } + cursor = page.list_complete ? null : page.cursor; + } while (cursor); + return entries; +} + +// intVar reads an integer variable with a default when it is unset or blank. +function intVar(env, name, fallback) { + const value = parseInt(env[name], 10); + return Number.isFinite(value) ? value : fallback; +} diff --git a/relay/test/index.js b/relay/test/index.js new file mode 100644 index 0000000000..7488863047 --- /dev/null +++ b/relay/test/index.js @@ -0,0 +1,12 @@ +// Node's test runner takes files and globs, not a directory, so the +// directory-form command `node --test test` resolves here; this entry loads +// every test file the same way the glob does. + +import { readdirSync } from 'node:fs'; + +const here = new URL('.', import.meta.url); +for (const name of readdirSync(here).sort()) { + if (name.endsWith('.test.js')) { + await import(new URL(name, here)); + } +} diff --git a/relay/test/index.test.js b/relay/test/index.test.js new file mode 100644 index 0000000000..8b5f992de3 --- /dev/null +++ b/relay/test/index.test.js @@ -0,0 +1,76 @@ +// index.test.js checks the document's shape, its key order and the signature. + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { buildIndex, signIndex } from '../src/index.js'; + +const CELLS = [ + { + metric: 'role_quality', role: 'worker', model: 'z-ai/glm-5.3-flash', + mean: 65.6, sd: 5.9, n: 62, installs: 4, + }, +]; + +test('buildIndex carries an integer version and keys in the fixed order', () => { + const text = buildIndex(CELLS, { + version: 1758117600, generated: '2026-09-17', minInstalls: 3, judges: ['j/x'], + }); + assert.doesNotMatch(text, /\n/); + const doc = JSON.parse(text); + assert.equal(doc.schema, 1); + assert.equal(doc.version, 1758117600); + assert.equal(Number.isInteger(doc.version), true); + assert.equal(doc.generated, '2026-09-17'); + assert.equal(doc.min_installs, 3); + assert.deepEqual(doc.judges, ['j/x']); + assert.deepEqual(doc.rubrics, { role_quality: 1 }); + assert.deepEqual(doc.aliases, {}); + assert.deepEqual(Object.keys(doc.metrics.role_quality), ['kind', 'unit', 'dims']); + assert.deepEqual(Object.keys(doc), [ + 'schema', 'version', 'generated', 'min_installs', 'judges', + 'rubrics', 'aliases', 'metrics', 'cells', + ]); + assert.deepEqual(Object.keys(doc.cells[0]), ['metric', 'role', 'model', 'mean', 'sd', 'n', 'installs']); +}); + +test('buildIndex with no cells still renders a document with an empty cells list', () => { + const doc = JSON.parse(buildIndex([], { + version: 1, generated: '2026-09-17', minInstalls: 3, judges: [], + })); + assert.deepEqual(doc.cells, []); +}); + +test('signIndex round-trips under the public key through crypto.subtle.verify', async () => { + const pair = await crypto.subtle.generateKey({ name: 'Ed25519' }, true, ['sign', 'verify']); + const priv = await crypto.subtle.exportKey('jwk', pair.privateKey); + const pub = await crypto.subtle.exportKey('jwk', pair.publicKey); + const seedB64 = fromBase64Url(priv.d); + const publicB64 = fromBase64Url(pub.x); + + const doc = buildIndex(CELLS, { + version: 7, generated: '2026-09-17', minInstalls: 3, judges: ['j/x'], + }); + const sigB64 = await signIndex(doc, seedB64, publicB64); + + const key = await crypto.subtle.importKey( + 'jwk', { kty: 'OKP', crv: 'Ed25519', x: pub.x }, { name: 'Ed25519' }, false, ['verify'], + ); + const sig = Uint8Array.from(atob(sigB64), (c) => c.charCodeAt(0)); + const ok = await crypto.subtle.verify( + { name: 'Ed25519' }, key, sig, new TextEncoder().encode(doc), + ); + assert.equal(ok, true); + + // The same document under a different byte does not verify. + const bad = await crypto.subtle.verify( + { name: 'Ed25519' }, key, sig, new TextEncoder().encode(doc + ' '), + ); + assert.equal(bad, false); +}); + +// fromBase64Url reassembles standard base64 from a JWK's base64url field. +function fromBase64Url(text) { + const base = text.replace(/-/g, '+').replace(/_/g, '/'); + return base + '='.repeat((4 - (base.length % 4)) % 4); +} diff --git a/relay/test/schema.test.js b/relay/test/schema.test.js new file mode 100644 index 0000000000..c999f9e31c --- /dev/null +++ b/relay/test/schema.test.js @@ -0,0 +1,92 @@ +// schema.test.js checks the row contract against the bytes the Go client +// writes (internal/pool/outbox, internal/pool/record). + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { validateInstall, validateRow } from '../src/schema.js'; + +// goodLine is one row exactly as the Go client would write it: compact JSON, +// the envelope's field order, and the record.Row payload inside. +function goodLine(overrides = {}) { + const payload = { + schema: 1, + metric: 'role_quality', + role: 'worker', + model: 'z-ai/glm-5.3-flash', + score: 65.6, + judge: 'anthropic/claude-opus-5', + door: 'task', + size: 'M', + day: '2026-09-17', + ...overrides.payload, + }; + const outer = { + schema: 1, + day: '2026-09-17', + nonce: '0123456789abcdef0123456789abcdef', + payload, + ...overrides.outer, + }; + return JSON.stringify(outer); +} + +const NOW = new Date('2026-09-17T12:00:00Z'); + +test('accepts the exact payload the Go side writes', () => { + const { row, error } = validateRow(goodLine(), NOW); + assert.equal(error, null); + assert.equal(row.day, '2026-09-17'); + assert.equal(row.nonce, '0123456789abcdef0123456789abcdef'); + assert.equal(row.payload.role, 'worker'); + assert.equal(row.payload.model, 'z-ai/glm-5.3-flash'); + assert.equal(row.payload.score, 65.6); + assert.equal(row.payload.judge, 'anthropic/claude-opus-5'); + assert.equal(row.payload.door, 'task'); + assert.equal(row.payload.size, 'M'); +}); + +test('accepts a day one day ahead but rejects two', () => { + assert.equal(validateRow(goodLine({ outer: { day: '2026-09-18' }, payload: { day: '2026-09-18' } }), NOW).error, null); + assert.notEqual(validateRow(goodLine({ outer: { day: '2026-09-19' }, payload: { day: '2026-09-19' } }), NOW).error, null); +}); + +test('rejects each broken envelope field', () => { + assert.notEqual(validateRow('', NOW).error, null); + assert.notEqual(validateRow('not json', NOW).error, null); + assert.notEqual(validateRow('[]', NOW).error, null); + assert.notEqual(validateRow(goodLine({ outer: { schema: 2 } }), NOW).error, null); + assert.notEqual(validateRow(goodLine({ outer: { day: '2026/09/17' } }), NOW).error, null); + assert.notEqual(validateRow(goodLine({ outer: { nonce: 'ABCDEF' } }), NOW).error, null); + assert.notEqual(validateRow(goodLine({ outer: { nonce: '0123456789abcdef0123456789abcde' } }), NOW).error, null); + assert.notEqual(validateRow(JSON.stringify({ schema: 1, day: '2026-09-17', nonce: '0123456789abcdef0123456789abcdef' }), NOW).error, null); + assert.notEqual(validateRow(JSON.stringify({ schema: 1, day: '2026-09-17', nonce: '0123456789abcdef0123456789abcdef', payload: [] }), NOW).error, null); +}); + +test('rejects each broken payload field', () => { + const bad = [ + { payload: { schema: 2 } }, + { payload: { metric: 'other' } }, + { payload: { role: 'reflex' } }, + { payload: { model: 'glm-5.3-flash' } }, + { payload: { model: 'z-ai/glm 5.3' } }, + { payload: { score: 101 } }, + { payload: { score: -1 } }, + { payload: { score: '65' } }, + { payload: { judge: 'opus' } }, + { payload: { door: 'cron' } }, + { payload: { size: 'XL' } }, + { payload: { day: 'yesterday' } }, + ]; + for (const override of bad) { + assert.notEqual(validateRow(goodLine(override), NOW).error, null, JSON.stringify(override)); + } +}); + +test('validateInstall takes 32 lowercase hex and nothing else', () => { + assert.equal(validateInstall('0123456789abcdef0123456789abcdef'), true); + assert.equal(validateInstall('0123456789ABCDEF0123456789abcdef'), false); + assert.equal(validateInstall('0123'), false); + assert.equal(validateInstall(undefined), false); + assert.equal(validateInstall('0123456789abcdef0123456789abcde'), false); +}); diff --git a/relay/test/sheet.test.js b/relay/test/sheet.test.js new file mode 100644 index 0000000000..11d64d6c07 --- /dev/null +++ b/relay/test/sheet.test.js @@ -0,0 +1,112 @@ +// sheet.test.js checks the running totals, the CRDT join and the aggregation. + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { aggregate, cellKey, fold, join, judgesOf } from '../src/sheet.js'; + +const KEY = cellKey({ + role: 'worker', + model: 'z-ai/glm-5.3-flash', + judge: 'anthropic/claude-opus-5', + door: 'task', + size: 'M', +}); + +function entry(install, day, key, mean, n = 4) { + return { install, day, key, triple: { n, s: mean * n, s2: mean * mean * n } }; +} + +test('cellKey keeps the five dimensions in order', () => { + assert.equal(KEY, 'worker|z-ai/glm-5.3-flash|anthropic/claude-opus-5|task|M'); +}); + +test('fold adds one score', () => { + assert.deepEqual(fold(null, 50), { n: 1, s: 50, s2: 2500 }); + assert.deepEqual(fold({ n: 1, s: 50, s2: 2500 }, 60), { n: 2, s: 110, s2: 6100 }); +}); + +test('join is commutative, associative, idempotent and takes the larger total', () => { + const a = { n: 2, s: 100, s2: 5200 }; + const b = { n: 3, s: 160, s2: 8600 }; + const c = { n: 1, s: 10, s2: 100 }; + assert.deepEqual(join(a, b), join(b, a)); + assert.deepEqual(join(join(a, b), c), join(a, join(b, c))); + assert.deepEqual(join(a, a), a); + assert.deepEqual(join(a, b), b); + assert.deepEqual(join(a, null), a); +}); + +test('aggregate takes n, installs and the n-weighted mean below five installs', () => { + const entries = [ + entry('i1', '2026-09-17', KEY, 50), + entry('i2', '2026-09-17', KEY, 60), + entry('i3', '2026-09-17', KEY, 70), + ]; + const cells = aggregate(entries, { minInstalls: 3 }); + assert.equal(cells.length, 1); + assert.equal(cells[0].role, 'worker'); + assert.equal(cells[0].model, 'z-ai/glm-5.3-flash'); + assert.equal(cells[0].installs, 3); + assert.equal(cells[0].n, 12); + assert.equal(cells[0].mean, 60); +}); + +test('aggregate pools the within-cell standard deviation from the triples', () => { + // Two installs, each four scores: the pooled sd is over all eight scores. + const entries = [ + { install: 'i1', day: '2026-09-17', key: KEY, triple: { n: 4, s: 200, s2: 10000 } }, // 50 ×4 + { install: 'i2', day: '2026-09-17', key: KEY, triple: { n: 4, s: 240, s2: 14400 } }, // 60 ×4 + ]; + const cells = aggregate(entries, { minInstalls: 3 }); + // sqrt((24400 - 440²/8) / 7) = sqrt((24400 - 24200)/7) = sqrt(200/7). + assert.ok(Math.abs(cells[0].sd - Math.sqrt(200 / 7)) < 1e-9); + assert.equal(cells[0].sd, Math.sqrt(200 / 7)); +}); + +test('aggregate keeps the Huber mean near six installs when one is absurd', () => { + const entries = [48, 50, 52, 49, 51, 50, 500].map((mean, i) => + entry(`i${i}`, '2026-09-17', KEY, mean)); + const cells = aggregate(entries, { minInstalls: 3 }); + assert.equal(cells[0].installs, 7); + assert.ok(Math.abs(cells[0].mean - 50) < 1, `mean ${cells[0].mean} should sit near 50`); +}); + +test('judge severity is removed: two judges offset by +10 agree on each model', () => { + const key = (model, judge) => cellKey({ + role: 'worker', model, judge, door: 'task', size: 'M', + }); + const ja = 'anthropic/claude-opus-5'; + const jb = 'deepseek/deepseek-v4.1-flash'; + const entries = [ + entry('a', '2026-09-17', key('z-ai/glm-5.3', ja), 50), + entry('b', '2026-09-17', key('z-ai/glm-5.3', jb), 60), // +10 severity + entry('a', '2026-09-17', key('z-ai/glm-5.3-flash', ja), 70), + entry('b', '2026-09-17', key('z-ai/glm-5.3-flash', jb), 80), // +10 severity + ]; + const cells = aggregate(entries, { minInstalls: 3 }); + const byModel = new Map(cells.map((c) => [c.model, c])); + // Both judges land on one mean per model: the +10 offset is gone. + const base = byModel.get('z-ai/glm-5.3').mean; + const flash = byModel.get('z-ai/glm-5.3-flash').mean; + assert.ok(Math.abs(base - 55) < 1e-9, `base ${base}`); + assert.ok(Math.abs(flash - 75) < 1e-9, `flash ${flash}`); + assert.ok(Math.abs((flash - base) - 20) < 1e-9); +}); + +test('aggregate publishes judges sorted and sorts below-minInstalls cells last', () => { + const thin = cellKey({ role: 'worker', model: 'a/one', judge: 'j/x', door: 'task', size: 'S' }); + const fat = cellKey({ role: 'worker', model: 'b/two', judge: 'j/y', door: 'do', size: 'L' }); + const entries = [ + entry('i1', '2026-09-17', thin, 90), + entry('i2', '2026-09-17', thin, 90), + entry('i1', '2026-09-17', fat, 10), + entry('i2', '2026-09-17', fat, 10), + entry('i3', '2026-09-17', fat, 10), + ]; + const cells = aggregate(entries, { minInstalls: 3 }); + const judges = judgesOf(entries); + assert.deepEqual(judges, ['j/x', 'j/y']); + assert.equal(cells[0].model, 'b/two'); // 3 installs meets the floor, sorts first + assert.equal(cells[1].model, 'a/one'); // 2 installs falls after +}); diff --git a/relay/test/worker.test.js b/relay/test/worker.test.js new file mode 100644 index 0000000000..a38b4bb770 --- /dev/null +++ b/relay/test/worker.test.js @@ -0,0 +1,270 @@ +// worker.test.js drives the Worker's routes against an in-memory KV stub. + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import worker from '../src/worker.js'; + +// KV is the smallest store the Worker needs: get, put and one list page. +// put honours an expirationTtl or expiration option the way the lock needs. +class KV { + constructor() { + this.map = new Map(); + this.expiry = new Map(); + } + async get(key) { + if (!this.map.has(key)) { + return null; + } + const until = this.expiry.get(key); + if (until !== undefined && Date.now() >= until) { + this.map.delete(key); + this.expiry.delete(key); + return null; + } + return this.map.get(key); + } + async put(key, value, options = {}) { + this.map.set(key, value); + if (Number.isFinite(options.expirationTtl)) { + this.expiry.set(key, Date.now() + options.expirationTtl * 1000); + } else if (Number.isFinite(options.expiration)) { + this.expiry.set(key, options.expiration * 1000); + } else { + this.expiry.delete(key); + } + } + async list({ prefix = '', cursor } = {}) { + void cursor; + const keys = [...this.map.keys()] + .filter((k) => k.startsWith(prefix)) + .sort() + .map((name) => ({ name })); + return { keys, list_complete: true, cursor: undefined }; + } +} + +const INSTALL = '0123456789abcdef0123456789abcdef'; + +function payload(overrides = {}) { + return { + schema: 1, metric: 'role_quality', role: 'worker', + model: 'z-ai/glm-5.3-flash', score: 50, + judge: 'anthropic/claude-opus-5', door: 'task', size: 'M', + day: '2026-09-17', ...overrides, + }; +} + +function line(overrides = {}) { + return JSON.stringify({ + schema: 1, day: '2026-09-17', nonce: '0123456789abcdef0123456789abcdef', + payload: payload(overrides), + }); +} + +// One Ed25519 pair for the whole file, exported as standard base64 the way the +// secret and the var are held. +const pair = await crypto.subtle.generateKey({ name: 'Ed25519' }, true, ['sign', 'verify']); +const privJwk = await crypto.subtle.exportKey('jwk', pair.privateKey); +const pubJwk = await crypto.subtle.exportKey('jwk', pair.publicKey); + +function env(extra = {}) { + return { + POOL: new KV(), + POOL_SIGNING_KEY: fromBase64Url(privJwk.d), + POOL_PUBLIC_KEY: fromBase64Url(pubJwk.x), + MIN_INSTALLS: '3', + ROWS_PER_INSTALL_PER_DAY: '500', + ...extra, + }; +} + +function fromBase64Url(text) { + const base = text.replace(/-/g, '+').replace(/_/g, '/'); + return base + '='.repeat((4 - (base.length % 4)) % 4); +} + +function post(body, headers = {}) { + return new Request('https://codeaf.agentfield.ai/pool/v1/rows', { + method: 'POST', headers, body, + }); +} + +// background collects the waitUntil promises a read schedules; drain awaits +// everything the reads scheduled, the way the platform settles them after +// the response. +function background() { + return { pending: [], waitUntil(promise) { this.pending.push(promise); } }; +} +async function drain(ctx) { + await Promise.all(ctx.pending.splice(0)); +} + +test('healthz answers ok and other paths answer an empty 404', async () => { + const e = env(); + const ok = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/healthz'), e); + assert.equal(ok.status, 200); + assert.equal(await ok.text(), 'ok'); + for (const path of ['/other', '/', '/pool', '/pool/nope']) { + const res = await worker.fetch(new Request(`https://codeaf.agentfield.ai${path}`), e); + assert.equal(res.status, 404, path); + assert.equal(await res.text(), ''); + } +}); + +test('submit requires the install header', async () => { + const res = await worker.fetch(post(line()), env()); + assert.equal(res.status, 400); +}); + +test('submit refuses a bad line, naming its number, and stores nothing', async () => { + const e = env(); + const body = line() + '\n' + '{"schema":1}' + '\n'; + const res = await worker.fetch(post(body, { 'X-Codeaf-Install': INSTALL }), e); + assert.equal(res.status, 400); + assert.match((await res.json()).error, /line 2/); + assert.equal(e.POOL.map.size, 0); +}); + +test('submit refuses an oversized body and too many lines', async () => { + const e = env(); + const big = await worker.fetch(post('x'.repeat((256 << 10) + 1), { 'X-Codeaf-Install': INSTALL }), e); + assert.equal(big.status, 413); + const many = line() + '\n'; + const body = many.repeat(201); + const res = await worker.fetch(post(body, { 'X-Codeaf-Install': INSTALL }), e); + assert.equal(res.status, 413); +}); + +test('submit folds a batch into sheets and answers 202', async () => { + const e = env(); + const body = line({ score: 50 }) + '\n' + line({ score: 60 }) + '\n'; + const res = await worker.fetch(post(body, { 'X-Codeaf-Install': INSTALL }), e); + assert.equal(res.status, 202); + assert.deepEqual(await res.json(), { accepted: 2 }); + const stored = JSON.parse(e.POOL.map.get( + `sheet/${INSTALL}/2026-09-17/worker|z-ai/glm-5.3-flash|anthropic/claude-opus-5|task|M`, + )); + assert.deepEqual(stored, { n: 2, s: 110, s2: 6100 }); + assert.equal(e.POOL.map.get(`quota/${INSTALL}/2026-09-17`), '2'); +}); + +test('submit refuses a batch past the daily quota and stores nothing', async () => { + const e = env({ ROWS_PER_INSTALL_PER_DAY: '1' }); + const body = line() + '\n' + line() + '\n'; + const res = await worker.fetch(post(body, { 'X-Codeaf-Install': INSTALL }), e); + assert.equal(res.status, 429); + assert.equal(e.POOL.map.size, 0); +}); + +test('the index is 404 before the first publication', async () => { + const e = env(); + const res = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json'), e); + assert.equal(res.status, 404); +}); + +test('scheduled publishes a signed index, served with a version ETag and 304', async () => { + const e = env(); + for (let i = 0; i < 3; i++) { + const name = `sheet/install${i}/2026-09-17/worker|z-ai/glm-5.3-flash|anthropic/claude-opus-5|task|M`; + await e.POOL.put(name, JSON.stringify({ n: 4, s: 200 + i * 4, s2: 10000 })); + } + await worker.scheduled({}, e); + + const docRes = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json'), e); + assert.equal(docRes.status, 200); + const etag = docRes.headers.get('ETag'); + assert.equal(docRes.headers.get('Cache-Control'), 'public, max-age=300'); + const doc = await docRes.text(); + const parsed = JSON.parse(doc); + assert.equal(Number.isInteger(parsed.version), true); + assert.equal(parsed.cells.length, 1); + assert.equal(parsed.cells[0].installs, 3); + + const sigRes = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json.sig'), e); + assert.equal(sigRes.status, 200); + const sig = Uint8Array.from(atob((await sigRes.text()).trim()), (c) => c.charCodeAt(0)); + const pub = await crypto.subtle.importKey( + 'jwk', { kty: 'OKP', crv: 'Ed25519', x: pubJwk.x }, { name: 'Ed25519' }, false, ['verify'], + ); + const verified = await crypto.subtle.verify( + { name: 'Ed25519' }, pub, sig, new TextEncoder().encode(doc), + ); + assert.equal(verified, true); + + const notModified = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json', { + headers: { 'If-None-Match': etag }, + }), e); + assert.equal(notModified.status, 304); +}); + +test('a read with nothing stored answers 404 once and the next read answers 200 after the background publish', async () => { + const e = env(); + const ctx = background(); + const first = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json'), e, ctx); + assert.equal(first.status, 404); + await drain(ctx); + const second = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json'), e); + assert.equal(second.status, 200); + assert.equal(JSON.parse(await second.text()).cells.length, 0); +}); + +test('a read of a document older than PUBLISH_EVERY serves the old document and republishes exactly once when two reads race', async () => { + const e = env(); + const old = Math.floor(Date.now() / 1000) - 7200; + const stale = JSON.stringify({ version: old, cells: [] }); + await e.POOL.put('index/doc', stale); + await e.POOL.put('index/sig', 'old-signature'); + await e.POOL.put('index/version', String(old)); + let publishes = 0; + const put = e.POOL.put.bind(e.POOL); + e.POOL.put = async (key, value, options) => { + if (key === 'index/doc') { + publishes++; + } + return put(key, value, options); + }; + const ctx = background(); + const url = 'https://codeaf.agentfield.ai/pool/index.json'; + const [a, b] = await Promise.all([ + worker.fetch(new Request(url), e, ctx), + worker.fetch(new Request(url), e, ctx), + ]); + assert.equal(a.status, 200); + assert.equal(b.status, 200); + assert.equal(await a.text(), stale); + assert.equal(await b.text(), stale); + await drain(ctx); + assert.equal(publishes, 1); + assert.notEqual(await e.POOL.get('index/publishing'), null); + const version = parseInt(await e.POOL.get('index/version'), 10); + assert.equal(version > old, true); +}); + +test('a read of a fresh document republishes nothing', async () => { + const e = env(); + await worker.scheduled({}, e); + const version = await e.POOL.get('index/version'); + const ctx = background(); + const res = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json'), e, ctx); + assert.equal(res.status, 200); + await drain(ctx); + assert.equal(ctx.pending.length, 0); + assert.equal(await e.POOL.get('index/version'), version); +}); + +test('scheduled still publishes', async () => { + const e = env(); + await worker.scheduled({}, e); + const res = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json'), e); + assert.equal(res.status, 200); + assert.equal(Number.isInteger(JSON.parse(await res.text()).version), true); +}); + +test('scheduled with no cells still publishes an empty index', async () => { + const e = env(); + await worker.scheduled({}, e); + const res = await worker.fetch(new Request('https://codeaf.agentfield.ai/pool/index.json'), e); + assert.equal(res.status, 200); + assert.deepEqual(JSON.parse(await res.text()).cells, []); +}); diff --git a/relay/tools/prime.py b/relay/tools/prime.py new file mode 100644 index 0000000000..530a70ebb7 --- /dev/null +++ b/relay/tools/prime.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +"""Prime the Model Pool relay from the scored runs the seed index already holds. + +This is a ONE-OFF. It is run once by the operator, over the private run ledgers +that never enter this repository, so that the first published index carries the +same cells the binary's embedded seed (internal/pool/index/seed.json) encodes. +Run it with --dry-run first: that prints the rows and checks them against the +seed without sending anything. + +It reads two newline-delimited JSON ledgers by path, rebuilds the ordinary Model +Pool rows the seat/score mapping implies, groups them per install and day, and +either prints the NDJSON (--dry-run) or POSTs each batch to /v1/rows. + +The ledgers stay outside the repository: pass their paths in on the command +line, and nothing here writes them back out. + + python3 prime.py --ledger --cells [--relay ] [--dry-run] +""" + +import argparse +import hashlib +import json +import os +import sys +import urllib.error +import urllib.request + +# The relay's submit route, the install header, and the content type the Go +# client (internal/pool/outbox) already sends. +ROWS_PATH = "/v1/rows" +INSTALL_HEADER = "X-Codeaf-Install" +CONTENT_TYPE = "application/x-ndjson" +DEFAULT_RELAY = "https://codeaf.agentfield.ai/pool" + +# The relay refuses a request with more than 200 lines and folds at most 500 +# rows into one install and one day (relay/src/worker.js). A day over the +# second number cannot be sent at all. +MAX_BATCH = 200 +ROWS_PER_INSTALL_PER_DAY = 500 + +# One judge id for every row: these runs were scored by review, not by a judge +# model, and one judge keeps the relay's judge-severity fit neutral. +JUDGE = "codeaf/reviewer" + +# The seed the self-check reads, relative to this file. +SEED_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..", + "internal", "pool", "index", "seed.json", +) + +# A run's worker seat names the worker by a short spelling on the ledger; these +# are the spellings the seed's cells were folded from. A worker whose name has +# no "/" after this aliasing is not a model id and its worker seat is skipped. +WORKER_ALIASES = { + "ds-v4.1-flash": "deepseek/deepseek-v4.1-flash", + "glm-5.3-flash": "z-ai/glm-5.3-flash", + "glm-5.3": "z-ai/glm-5.3", +} + +# A task-door crew names its checker by a short spelling after "task+"; these +# are the spellings the seed's high cells were folded from. +HIGH_ALIASES = { + "kimi-k3": "moonshotai/kimi-k3", + "qwen3.8-max": "qwen/qwen3.8-max-0902", + "qwen3.8-max-0902": "qwen/qwen3.8-max-0902", + "claude-fable-5.1": "anthropic/claude-fable-5.1", + "fable-5.1": "anthropic/claude-fable-5.1", + "claude-opus-5": "anthropic/claude-opus-5", + "opus-5": "anthropic/claude-opus-5", +} + + +def day_of(at): + """The YYYY-MM-DD date of a ledger "at" stamp ("…T…" or "… …").""" + text = str(at or "").strip() + day = text[:10] + return day if len(day) == 10 and day[4] == "-" and day[7] == "-" else "" + + +def install_of(session): + """The install id a session submits under: the first 32 hex of its sha256. + + One install per originating session, and the same 32 characters the + X-Codeaf-Install header carries, so the relay folds one session's runs + under one install. + """ + return hashlib.sha256(str(session).encode("utf-8")).hexdigest()[:32] + + +def as_int(value, default=0): + try: + return int(value) + except (TypeError, ValueError): + return default + + +def is_no_work(rating, notes): + """A run is "no work" when its rating is at most 1 and its notes say so. + + Such a run is a FAILURE for the worker seat and contributes nothing to any + other seat. + """ + try: + low = float(rating) <= 1 + except (TypeError, ValueError): + low = False + return low and "no work" in str(notes or "").lower() + + +def score_of(major, no_work): + """The 0/100 score a seat takes from a run's outcome.""" + if no_work: + return 0 + return 100 if as_int(major) == 0 else 0 + + +def resolve_worker(name): + return WORKER_ALIASES.get(str(name or "").strip(), str(name or "").strip()) + + +def resolve_high(name): + return HIGH_ALIASES.get(str(name or "").strip(), str(name or "").strip()) + + +def checker_of(crew): + """The checker short name a task-door crew names. + + A crew is "task+" or "task+learn(/)"; the checker + is the whole tail, or the part after the slash inside "learn(...)". + """ + rest = crew[len("task+"):] + if rest.startswith("learn(") and rest.endswith(")"): + inner = rest[len("learn("):-1] + if "/" in inner: + return inner.rsplit("/", 1)[1].strip() + return inner.strip() + return rest.strip() + + +def size_from_task(task): + """A ledger row's size: the leading "S "/"M "/"L " or "S-"/"M-"/"L-", else M.""" + text = str(task or "") + if len(text) >= 2 and text[0] in "SML" and text[1] in (" ", "-"): + return text[0] + return "M" + + +def read_jsonl(path): + """Every non-blank line of a JSONL file, parsed.""" + rows = [] + with open(path, "r", encoding="utf-8") as handle: + for number, line in enumerate(handle, 1): + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError as err: + raise SystemExit("%s:%d: not json: %s" % (path, number, err)) + return rows + + +def load_ledger(path): + """Ledger rows deduplicated by (session, run), last wins. + + A row with tag == "bench" or a truthy bench field is an experiment arm and + is dropped entirely. + """ + latest = {} + for row in read_jsonl(path): + if row.get("tag") == "bench" or row.get("bench"): + continue + latest[(str(row.get("session")), str(row.get("run")))] = row + return list(latest.values()) + + +def load_cells(path): + """The completed cells of a cells ledger: a lease joined to its done row. + + done rows are deduplicated by id, last wins. A cell whose id equals a + ledger run is skipped later, where the ledger's own run is known. + """ + leases = {} + dones = {} + for row in read_jsonl(path): + if row.get("kind") == "lease": + leases[str(row.get("id"))] = row + elif row.get("kind") == "done": + dones[str(row.get("id"))] = row + return leases, dones + + +def attempts_from_ledger(path): + """One attempt per ledger run.""" + attempts = [] + for row in load_ledger(path): + crew = str(row.get("crew") or "") + attempts.append({ + "session": row.get("session"), + "crew": crew, + "worker": row.get("worker"), + "planner": row.get("planner"), + "major": row.get("major"), + "rating": row.get("rating"), + "notes": row.get("notes"), + "day": day_of(row.get("at")), + "size": size_from_task(row.get("task")), + "door": "do" if crew == "do" else "task", + }) + return attempts + + +def attempts_from_cells(path, ledger_runs): + """One attempt per completed cell ledger row. + + The crew is "task+" on the task door and "do" otherwise; a cell + whose id is also a ledger run is skipped, because the ledger already holds + that run. + """ + leases, dones = load_cells(path) + attempts = [] + for cell_id, lease in leases.items(): + if cell_id in ledger_runs: + continue + done = dones.get(cell_id) + if done is None: + continue + door = str(lease.get("door") or "") + if door == "task": + crew = "task+" + str(lease.get("checker") or "") + else: + crew = "do" + size = str(lease.get("size") or "M") + attempts.append({ + "session": lease.get("session"), + "crew": crew, + "worker": lease.get("worker"), + "planner": lease.get("thinker"), + "major": done.get("major"), + "rating": done.get("rating"), + # A done row says "no work" with a flag; fold it into the notes so + # the one rule in is_no_work reads both ledgers the same way. + "notes": ("no work; " if done.get("nowork") else "") + str(done.get("notes") or ""), + "day": day_of(lease.get("at")), + "size": size if size in ("S", "M", "L") else "M", + "door": "do" if crew == "do" else "task", + }) + return attempts + + +def rows_of(attempt): + """The Model Pool rows one attempt contributes: one per seat it scores. + + The worker seat is always scored (when its name is a model id); a task-door + crew adds the high seat, any other crew the mastermind seat. A "no work" run + is a worker failure and scores no other seat; a mastermind whose model is + not a model id, or is the worker itself, is dropped. + """ + no_work = is_no_work(attempt["rating"], attempt["notes"]) + score = score_of(attempt["major"], no_work) + worker = resolve_worker(attempt["worker"]) + install = install_of(attempt["session"]) + day = attempt["day"] + + seats = [] + if "/" in worker: + seats.append(("worker", worker)) + if not no_work: + if attempt["crew"].startswith("task+"): + checker = resolve_high(checker_of(attempt["crew"])) + if "/" in checker: + seats.append(("high", checker)) + else: + planner = str(attempt["planner"] or "").strip() + if "/" in planner and planner != worker: + seats.append(("mastermind", planner)) + + rows = [] + for role, model in seats: + rows.append({ + "schema": 1, + "day": day, + "nonce": install, + "payload": { + "schema": 1, + "metric": "role_quality", + "role": role, + "model": model, + "score": score, + "judge": JUDGE, + "door": attempt["door"], + "size": attempt["size"], + "day": day, + }, + }) + return rows + + +def build_rows(ledger_path, cells_path): + """Every row both ledgers imply, in ledger-then-cells order.""" + ledger_rows = load_ledger(ledger_path) + ledger_runs = {str(row.get("run")) for row in ledger_rows} + attempts = attempts_from_ledger(ledger_path) + attempts += attempts_from_cells(cells_path, ledger_runs) + rows = [] + for attempt in attempts: + rows.extend(rows_of(attempt)) + return rows + + +def cell_totals(rows): + """Per (role, model): how many rows and how many scored 100.""" + totals = {} + for row in rows: + payload = row["payload"] + key = (payload["role"], payload["model"]) + count, ok = totals.get(key, (0, 0)) + totals[key] = (count + 1, ok + (1 if payload["score"] == 100 else 0)) + return totals + + +def load_seed(path): + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def posterior(ok, n): + """The seed's Beta(1,1) posterior mean of a cell: 100·(1+ok)/(2+n).""" + return 100.0 * (1 + ok) / (2 + n) + + +def report_totals(rows, out): + """Per install: rows and distinct days. Per (role, model): count and mean.""" + by_install = {} + for row in rows: + install = row["nonce"] + by_install.setdefault(install, set()).add(row["day"]) + out.write("installs:\n") + for install in sorted(by_install): + count = sum(1 for row in rows if row["nonce"] == install) + out.write(" %s rows=%d days=%d\n" % (install, count, len(by_install[install]))) + out.write("cells (role, model: count, mean score):\n") + totals = cell_totals(rows) + for (role, model) in sorted(totals): + count, ok = totals[(role, model)] + out.write(" %-10s %-28s n=%-4d mean=%.1f\n" + % (role, model, count, 100.0 * ok / count)) + + +def self_check(rows, seed, out): + """Compare our cells against the seed's and fail when an n differs. + + Every seed cell with n >= 3 must be reproduced exactly, and the seed's mean + must be the Beta(1,1) posterior of the same ok/n. + """ + totals = cell_totals(rows) + expected = seed.get("cells", []) + out.write("seed self-check (every seed cell with n >= 3 must match):\n") + out.write(" %-10s %-28s %6s %6s %6s | %6s %9s %9s\n" + % ("role", "model", "ours n", "ours ok", "ours P", "seed n", "seed mean", "posterior")) + mismatches = [] + for cell in expected: + role = str(cell.get("role")) + model = str(cell.get("model")) + seed_n = int(cell.get("n") or 0) + ours_n, ours_ok = totals.get((role, model), (0, 0)) + ours_p = (ours_ok / ours_n) if ours_n else 0.0 + out.write(" %-10s %-28s %6d %6d %6.3f | %6d %9.1f %9.1f\n" + % (role, model, ours_n, ours_ok, ours_p, seed_n, + float(cell.get("mean") or 0.0), posterior(ours_ok, ours_n))) + if seed_n >= 3 and ours_n != seed_n: + mismatches.append("%s/%s: seed n=%d but we built n=%d" + % (role, model, seed_n, ours_n)) + return mismatches + + +def batches_of(rows): + """Rows grouped by (install, day), each day split into batches of at most 200. + + A day that holds more than the relay's per-install per-day quota at all is + an error: it cannot be submitted legally. + """ + groups = {} + for row in rows: + groups.setdefault((row["nonce"], row["day"]), []).append(row) + batches = [] + for key in sorted(groups): + group = groups[key] + if len(group) > ROWS_PER_INSTALL_PER_DAY: + raise SystemExit( + "install %s on %s holds %d rows, over the %d-per-day quota; " + "the relay would refuse it" % (key[0], key[1], len(group), + ROWS_PER_INSTALL_PER_DAY)) + for start in range(0, len(group), MAX_BATCH): + batches.append((key[0], group[start:start + MAX_BATCH])) + return batches + + +def ndjson(rows): + return "\n".join(json.dumps(row, separators=(",", ":")) for row in rows) + "\n" + + +def post(relay, install, body): + """One batch to /v1/rows, answering the HTTP status.""" + url = relay.rstrip("/") + ROWS_PATH + request = urllib.request.Request( + url, + data=body.encode("utf-8"), + method="POST", + # The edge in front of the relay refuses the urllib default agent + # outright (403, before the Worker runs); name the tool instead. + headers={INSTALL_HEADER: install, "Content-Type": CONTENT_TYPE, + "User-Agent": "codeaf-prime/1"}, + ) + try: + with urllib.request.urlopen(request) as response: + return response.status + except urllib.error.HTTPError as err: + return err.code + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--ledger", required=True, help="path to the ledger.jsonl of scored runs") + parser.add_argument("--cells", required=True, help="path to the cells.jsonl of leases and done rows") + parser.add_argument("--relay", default=DEFAULT_RELAY, + help="the relay base URL (default: %(default)s)") + parser.add_argument("--dry-run", action="store_true", + help="print the rows and check them against the seed; send nothing") + args = parser.parse_args(argv) + + rows = build_rows(args.ledger, args.cells) + + if args.dry_run: + sys.stdout.write(ndjson(rows)) + report_totals(rows, sys.stderr) + mismatches = self_check(rows, load_seed(SEED_PATH), sys.stderr) + if mismatches: + for line in mismatches: + sys.stderr.write("mismatch: %s\n" % line) + return 1 + return 0 + + failed = False + for install, group in batches_of(rows): + status = post(args.relay, install, ndjson(group)) + print("%s %s: %d rows -> %d" % (install, group[0]["day"], len(group), status)) + # The relay accepts a batch with 202 (relay/src/worker.js handleRows), + # so any 2xx is a batch that arrived; 400 or 429 is a refusal. + if not 200 <= status < 300: + failed = True + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/relay/verify.mjs b/relay/verify.mjs new file mode 100644 index 0000000000..87d6eff610 --- /dev/null +++ b/relay/verify.mjs @@ -0,0 +1,45 @@ +#!/usr/bin/env node +// Verifies the detached Ed25519 signature over a Model Pool index document. +// +// node relay/verify.mjs index.json index.json.sig +// +// The signature is standard base64; whitespace around it is ignored. Exit 0 +// with no output when it verifies over the exact bytes of the document file. +// Exit 1, naming the failure, when the signature does not verify, when an +// argument is missing, or when a file cannot be read. + +import { readFileSync } from 'node:fs'; + +const [docPath, sigPath, publicB64] = process.argv.slice(2); +if (!docPath || !sigPath || !publicB64) { + console.error('usage: node relay/verify.mjs index.json index.json.sig '); + process.exit(1); +} + +let docBytes, signature; +try { + docBytes = readFileSync(docPath); + signature = Buffer.from(readFileSync(sigPath, 'utf8').trim(), 'base64'); +} catch (err) { + console.error(`cannot read input: ${err.message}`); + process.exit(1); +} + +let ok; +try { + const jwk = { + kty: 'OKP', + crv: 'Ed25519', + x: Buffer.from(publicB64, 'base64').toString('base64url'), + }; + const key = await globalThis.crypto.subtle.importKey('jwk', jwk, { name: 'Ed25519' }, false, ['verify']); + ok = await globalThis.crypto.subtle.verify({ name: 'Ed25519' }, key, signature, docBytes); +} catch (err) { + console.error(`cannot verify: ${err.message}`); + process.exit(1); +} + +if (!ok) { + console.error('signature does not verify'); + process.exit(1); +} diff --git a/relay/wrangler.toml b/relay/wrangler.toml new file mode 100644 index 0000000000..2b208ca5c7 --- /dev/null +++ b/relay/wrangler.toml @@ -0,0 +1,43 @@ +name = "codeaf-pool" +main = "src/worker.js" +compatibility_date = "2026-09-01" + +# A route on the path prefix, not a custom domain: the Worker owns only +# codeaf.agentfield.ai/pool/* and the host can carry other services. +routes = [ + { pattern = "codeaf.agentfield.ai/pool/*", zone_name = "agentfield.ai" } +] + +[[kv_namespaces]] +binding = "POOL" +# The runbook fills this id: `cd relay && wrangler kv namespace create POOL`. +id = "4b5cf70bed444ceb84a23ebf2be692e9" + +# Observability as set in the dashboard, kept here so a redeploy does not +# silently change it. The Worker itself logs nothing; what these persist are +# the platform's own invocation records (route, status, timing, request +# metadata). Turn persistence off, or lower the sampling, for a relay that +# should keep no record of who called it. +[observability] +[observability.logs] +enabled = true +head_sampling_rate = 1 +invocation_logs = true +persist = true +[observability.traces] +enabled = true +head_sampling_rate = 1 +persist = true + +[triggers] +crons = ["17 * * * *"] + +[vars] +# The public half of the index signing key built into the binary. The private +# half is the POOL_SIGNING_KEY secret, set by the runbook and never stored here. +POOL_PUBLIC_KEY = "WOAo+g/oKxAV9vVqv2Q14w1TyyyiouwFO2fC0zgcps0=" +MIN_INSTALLS = "3" +ROWS_PER_INSTALL_PER_DAY = "500" +# How long a published index stays fresh: the first read past this age +# republishes in the background and serves what is stored meanwhile. +PUBLISH_EVERY = "3600"