diff --git a/.claude/skills/build-s1a-agent/SKILL.md b/.claude/skills/build-s1a-agent/SKILL.md index 35f6fa3..56cbc97 100644 --- a/.claude/skills/build-s1a-agent/SKILL.md +++ b/.claude/skills/build-s1a-agent/SKILL.md @@ -1,6 +1,6 @@ --- name: build-s1a-agent -description: Build a new System 1 agent (an openJiuwen agent with a System 1 decision model in its model slot) for a task the user names, in this repository. Runs a fit probe first, then scaffolds one agent module from the template of the right front (tool, browser or rail), its test and its README row, and verifies it slot by slot. Use when the user asks to build, add or scaffold a System 1 agent, an S1A agent, a Jev agent, a new game, task, site or rail for s1a. +description: Build a new System 1 agent (an openJiuwen agent with a System 1 decision model in its model slot) for a task the user names, in this repository. Runs a fit probe first, then scaffolds one agent module from the template of the right front (tool, browser or rail), its test and its README row, and verifies it model by model. Use when the user asks to build, add or scaffold a System 1 agent, an S1A agent, a Jev agent, a new game, task, site or rail for s1a. --- # Build a System 1 agent @@ -47,7 +47,7 @@ Copy the front's template from `s1a/agents/_templates/` to `s1a/agents/.py part; the template's comments say what each part is. `SPEC.name` is the module name. Write `tests/test_agents_.py` with the two tool-front classes of `tests/test_templates.py`, `TestNimEnv` and `TestNimThroughTheLoop`, rewritten for the new module: the env's reset, candidates and winning line, then the rule -and random slots through `series.play` with `loop.WORKSPACE` and `series.optional_chat_model` patched as there. +and random models through `series.play` with `loop.WORKSPACE` and `series.optional_chat_model` patched as there. For a browser agent, copy `TestBrowserTemplateOffline`: the spec reaches the faked subagent through `support.browse_offline`. For a rail, copy `TestRailTemplateOffline`: precision and recall on five hand-labelled records through a `ScriptedModel(noul=[...])` from `s1a.decision_models`. Add one row to the agents table in `README.md`. Follow `references/state-design.md` for the @@ -60,20 +60,20 @@ Run each command, read its output, fix the agent before the next rung. Stop at t ```bash uv run pytest tests/test_agents_.py -q # the adapter contract, no keys -uv run s1a run --slot random --rethink off --episodes 3 # mechanics through the loop, no keys -uv run s1a run --slot rule --rethink off --episodes 3 # when a baseline exists -uv run s1a run --slot jev --rethink off --episodes 3 --log # keys: latency, invalid keys must be 0 -uv run s1a run --slot llm --rethink off --episodes 3 # the same seeds with the chat model -uv run python -m evals.table evals/results # one row per slot +uv run s1a run --model random --rethink off --episodes 3 # mechanics through the loop, no keys +uv run s1a run --model rule --rethink off --episodes 3 # when a baseline exists +uv run s1a run --model jev --rethink off --episodes 3 --log # keys: latency, invalid keys must be 0 +uv run s1a run --model llm --rethink off --episodes 3 # the same seeds with the chat model +uv run python -m evals.table evals/results # one row per model uv run pytest tests -q # the whole suite stays green ``` Every `run` prints one JSON object: the series summary with `scored`, the episodes that got a score, `errors`, the -count the model in the slot could not play, and `job_dir`. The `random` and `rule` slots exist for the tool front only; `laya` -(Laya in process, after `uv sync --extra laya`) fills any slot `jev` does, and so does `cua` (Cua-S1 Nano, after +count the model could not play, and `job_dir`. `random` and `rule` exist for the tool front only; `laya` +(Laya in process, after `uv sync --extra laya`) runs wherever `jev` does, and so does `cua` (Cua-S1 Nano, after `uv sync --extra cua`) except on a rail. For a browser or rail agent the key-free rung is the offline test from step 4. The paid rung follows. A browser agent runs one task per call and -needs the chat-model key and a Jev key: `uv run s1a run --slot jev --goal "..."`. A rail runs its +needs the chat-model key and a Jev key: `uv run s1a run --model jev --goal "..."`. A rail runs its labelled set and needs a Jev key: `uv run s1a run --labelled-set records.jsonl`. Report the table and stop. Series of a hundred episodes cost money; ask the user before starting one. diff --git a/.claude/skills/build-s1a-agent/references/fronts.md b/.claude/skills/build-s1a-agent/references/fronts.md index e8989dd..2568fb5 100644 --- a/.claude/skills/build-s1a-agent/references/fronts.md +++ b/.claude/skills/build-s1a-agent/references/fronts.md @@ -5,13 +5,13 @@ Every agent states every field. The spec classes in `s1a/spec.py` define no defa ## Tool front: `ToolAgentSpec`, template `_templates/tool_agent.py` A DeepAgent plays through two tools, `observe` and `act`, with a System 1 decision model in the model slot -(`--slot jev|llm|random|rule|laya|cua`). +(`--model jev|llm|random|rule|laya|cua`). | field | meaning | |---|---| | `name` | the module name: lowercase letters, digits and `_` after a letter; also the job folder name | | `description` | one sentence, shown by `list_agents` and the caller skill | -| `rules` | the text the model in the slot reads on every decision | +| `rules` | the text the model reads on every decision | | `budget` | `Budget(max_steps, timeout_s, stall_after)`; the defaults of `--max-steps` and `--timeout` | | `flags` | `(ArgumentParser) -> None`: the agent's own switches, after the shared ones | | `series` | `(Namespace) -> Series`: the seeds, `env_for(seed)`, the page `session`, the `baseline`, `annotate` | @@ -24,7 +24,7 @@ window's clickable elements as candidates, plus `done` and `abstain`, and `s1a/d ## Browser front: `BrowserAgentSpec`, template `_templates/browser_agent.py` -openJiuwen's browser subagent with `BrowserDecisionModel` in the slot (`--slot jev` for TypeSafe Jev, `laya` for +openJiuwen's browser subagent with `BrowserDecisionModel` in the slot (`--model jev` for TypeSafe Jev, `laya` for Laya in process, `cua` for Cua-S1 Nano in process, `llm` for the chat model alone); the chat model types values and writes the answer. | field | meaning | diff --git a/.claude/skills/build-s1a-agent/references/state-design.md b/.claude/skills/build-s1a-agent/references/state-design.md index b48910b..c12c6a4 100644 --- a/.claude/skills/build-s1a-agent/references/state-design.md +++ b/.claude/skills/build-s1a-agent/references/state-design.md @@ -22,16 +22,16 @@ What the loop sends Jev on every decision: the observation, the candidates, the - Under about 120 words. Facts to recognise, phrased as what a good move looks like. Leave out chains of reasoning. - Name the winning shapes and the traps in the state's own words so that Jev can match them by recognition. -- Every slot reads the same text, including the chat-model arm. +- Every model reads the same text, including the chat-model arm. ## The budget and the baseline -- `max_steps`: the acts one episode may spend; the chat slot gets twice as many iterations for malformed calls. +- `max_steps`: the acts one episode may spend; the chat model gets twice as many iterations for malformed calls. - `timeout_s`: the wall clock per episode; a timed-out episode keeps its score so far. - `stall_after`: acts without a score change before the rethink rail asks the chat model for a plan; 0 for games where every act changes the score or the question. - Give a `baseline` whenever a rule or an expert plan is known; it is the upper or lower bound the table shows. -- The `random` slot draws from its own stream, derived from the episode seed. An env opponent seeded from the plain +- The `random` model draws from its own stream, derived from the episode seed. An env opponent seeded from the plain integer seed, as the template does, draws a different stream. - In the summary a score above 0 counts as a win and below 0 as a loss. A draw scored 0.5 counts as a win there. diff --git a/.env.example b/.env.example index bca2d40..58acac1 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ # Jev: a direct TypeSafe key. Leave TYPESAFE_API_URL unset with it; setting that URL selects the OpenRouter proxy below. TYPESAFE_API_KEY= -# The chat model (typed values, final answers, the llm slot, the rethink planner); every slot that calls it needs MODEL_NAME. +# The chat model (typed values, final answers, `--model llm`, the rethink planner); every model that calls it needs MODEL_NAME. # OPENROUTER_* stands in for LLM_API_KEY and LLM_BASE_URL when those are unset. OPENROUTER_API_KEY= OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 @@ -27,14 +27,14 @@ MODEL_NAME=google/gemini-2.5-flash # HF_HOME=~/.cache/huggingface # where the laya and cua checkpoints download on first use # HF_HUB_OFFLINE=1 # after the first run, no network for the checkpoints -# ---- Laya (the in-process decision model behind --slot laya; needs `uv sync --extra laya`) ---- +# ---- Laya (the in-process decision model behind --model laya; needs `uv sync --extra laya`) ---- # LAYA_MODEL=convaiinnovations/laya # LAYA_SUBFOLDER=multilingual # or typed-decisions # LAYA_DEVICE=cpu # cuda when available # LAYA_MAX_LEN=1024 # LAYA_HEAD_MAX_LEN=512 # raise for choice questions with many options -# ---- Cua-S1 Nano (the in-process option scorer behind --slot cua; needs `uv sync --extra cua`) ---- +# ---- Cua-S1 Nano (the in-process option scorer behind --model cua; needs `uv sync --extra cua`) ---- # CUA_S1_CHECKPOINT=cua-ai/cua-s1-nano-0.1 # a Hugging Face id, or a local directory holding / # CUA_S1_SUBFOLDER=text # the text-only checkpoint; the window is 256 bytes of state # CUA_S1_DEVICE=auto # cpu, cuda, mps diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6f9d87d..e9dd956 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -22,5 +22,5 @@ uv run s1a ... - OS and Python version: - `uv sync` extras installed (`blackjack`, `alfworld`, `laya`, `cua`, ...): -- slot (`jev`, `laya`, `cua`, `llm`) and, for `jev`, direct key or OpenRouter: +- `--model` (`jev`, `laya`, `cua`, `llm`) and, for `jev`, direct key or OpenRouter: - commit (`git rev-parse --short HEAD`): diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 76bd078..450c3a7 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,6 +1,6 @@ --- name: Feature request -about: A new agent, slot, rail or eval +about: A new agent, decision model, rail or eval labels: enhancement --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 0907cba..bb80f85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow semver. +## Unreleased + +### Changed + +- `--model` picks the model on every agent, on `decide` and on `probe`: `jev`, `laya`, `cua`, `llm`, `random` or + `rule`. The results table's column, the replay page's badge data and a browser run's `answer.json` name it + `model` as well; the replay still reads the `slot` key of records written by 0.1.0. + ## 0.1.0 - 2026-09-23 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9633095..2107ee6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,8 +66,8 @@ Everything outside `openjiuwen` is an extra. An agent whose extra is missing say | `alfworld` | alfworld, textworld | `s1a run alfworld`; also `ALFWORLD_DATA` and Python 3.11, see `evals/README.md` | | `alfworld-visual` | the `alfworld` extra, ai2thor 2.1.0, torch | `evals/replay/thor_replay.py`, the AI2-THOR scene behind an ALFWorld trial in the replay page; the 400 MB Unity build downloads on first use | | `report` | pillow, playwright | `python -m evals.replay`, the showcase pages and GIFs; `--gif` also needs `uv run playwright install chromium` | -| `laya` | laya (torch, transformers) | `--slot laya` on every agent and on `decide` and `probe`: Laya in process, no Jev key; the checkpoint downloads into the Hugging Face cache (`HF_HOME`) on first use | -| `cua` | cua-s1 (torch), huggingface-hub | `--slot cua` on tool and browser agents and on `decide` and `probe`: Cua-S1 Nano in process; the 3 MB checkpoint downloads into the Hugging Face cache (`HF_HOME`) on first use | +| `laya` | laya (torch, transformers) | `--model laya` on every agent and on `decide` and `probe`: Laya in process, no Jev key; the checkpoint downloads into the Hugging Face cache (`HF_HOME`) on first use | +| `cua` | cua-s1 (torch), huggingface-hub | `--model cua` on tool and browser agents and on `decide` and `probe`: Cua-S1 Nano in process; the 3 MB checkpoint downloads into the Hugging Face cache (`HF_HOME`) on first use | | `dev` | pytest, pytest-asyncio, ruff, ty | the test suite, `scripts/smoke.sh` and the lint and type checks | `uv sync --all-extras` installs all seven. The CLI runs from a checkout; a wheel install (`uv tool install`, @@ -87,7 +87,7 @@ The `desktop` agent runs on Windows or macOS and has no extra. It needs Cua Driv `.env` needs a key for Jev (`TYPESAFE_API_KEY` direct, or `OPENROUTER_API_KEY` for the proxy) and for the chat model (`OPENAI_API_KEY` or `LLM_API_KEY`, `MODEL_NAME`; `OPENROUTER_API_KEY` and `OPENROUTER_BASE_URL` stand in -for `LLM_*` when those are unset). Exported variables win over the file. `--slot laya` and `--slot cua` need no +for `LLM_*` when those are unset). Exported variables win over the file. `--model laya` and `--model cua` need no decision key. A host agent (Claude Code, Codex, Cursor, Hermes) reaches the keys through its own environment or the checkout's `.env`. diff --git a/README.md b/README.md index e136759..5320a07 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ † The first Allrecipes task of the [WebVoyager](https://github.com/MinorJerry/WebVoyager) task set ([He et al., 2024](https://arxiv.org/abs/2401.13919), Apache-2.0, attribution in [NOTICE](NOTICE)): a vegetarian lasagna with over 100 reviews, 4.5 stars or more, for 6. The chat model of that row is Claude Fable 5.1 through -OpenRouter; both slots pay it for the typed search text and the answer. \* Estimated; the chat-model run recorded no +OpenRouter; both models pay it for the typed search text and the answer. \* Estimated; the chat-model run recorded no cost. Each replay below is the episode behind its row, Jev on the left and the chat model on the right, both on the wall clock. The other Allrecipes runs, longer games and the Google Flights driver comparison: [docs/benchmarks.md](docs/benchmarks.md). @@ -82,18 +82,18 @@ uv sync && cp .env.example .env # the first sync resolves the openjiuwen pin ``` Put a Jev key in `.env` (`TYPESAFE_API_KEY` from the [TypeSafe console](https://console.typesafe.ai), or -`OPENROUTER_API_KEY`), then ask for one decision and run one agent on both slots: +`OPENROUTER_API_KEY`), then ask for one decision and run one agent with each model: ```bash uv run s1a decide --state '{"player_total": 18, "dealer_upcard": 9}' \ --option hit="take a card" --option stand="keep the hand" --rules "stand on 17 or more" uv sync --extra blackjack -uv run s1a run blackjack --slot jev --rethink off --episodes 20 -uv run s1a run blackjack --slot llm --rethink off --episodes 20 # the chat model in the same slot +uv run s1a run blackjack --model jev --rethink off --episodes 20 +uv run s1a run blackjack --model llm --rethink off --episodes 20 # the chat model in the same agent ``` `decide` prints one JSON object with `choice`, a probability per option, `confidence` and `ms`; `run` writes a job -folder with the score. Without a key, `--slot cua` answers in process after `uv sync --extra cua`. +folder with the score. Without a key, `--model cua` answers in process after `uv sync --extra cua`. ### As an MCP server @@ -138,7 +138,7 @@ interface fits: [docs/architecture.md](docs/architecture.md), [docs/decision-mod - [docs/benchmarks.md](docs/benchmarks.md): the six runs above, the Google Flights driver comparison, a longer game, the guard rail. - [docs/skills.md](docs/skills.md): the caller skill, the builder skill, what to delegate. - [docs/agents.md](docs/agents.md): every agent with its flags, run command and extra. -- [docs/architecture.md](docs/architecture.md) and [docs/decision-models.md](docs/decision-models.md): the fronts, the slot, the model interface, adding a backend. +- [docs/architecture.md](docs/architecture.md) and [docs/decision-models.md](docs/decision-models.md): the fronts, the model slot, the model interface, adding a backend. - [docs/browser-front.md](docs/browser-front.md): the browser policy, decision by decision. - [docs/why.md](docs/why.md): the problem, the philosophy, the precedents. - [docs/roadmap.md](docs/roadmap.md) and [CHANGELOG.md](CHANGELOG.md). diff --git a/agents/s1a-browser.md b/agents/s1a-browser.md index c895fda..facc39b 100644 --- a/agents/s1a-browser.md +++ b/agents/s1a-browser.md @@ -10,7 +10,7 @@ You run one page task through the s1a command and report the result. 1. Turn the request into one goal sentence: the site URL first, the values to enter, and the stop condition ("Stop when the matching results are visible"). -2. Run `uv run --project ${CLAUDE_PLUGIN_ROOT} s1a run flights --slot jev --goal ""`. +2. Run `uv run --project ${CLAUDE_PLUGIN_ROOT} s1a run flights --model jev --goal ""`. 3. Read the JSON object on stdout. When `ok` is true, answer with `final` and name `terminal.url` and `terminal.title`. When `ok` is false, report `error`, then `status` and the last actions in `report.history` when present (a timeout leaves `error` only). A BLOCKED or timed-out run exits 0 with `ok` false. Exit 1 with diff --git a/docs/agents.md b/docs/agents.md index e6e471e..c7ff39a 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -1,30 +1,30 @@ # Agents: what each one measures and how to run it -Four kinds of agent share the loop, the slots and the job folders: browser use over `@playwright/mcp`, computer use +Four kinds of agent share the loop, the models and the job folders: browser use over `@playwright/mcp`, computer use over [Cua Driver](https://cua.ai/docs/cua-driver), games and embodied text, and rails that answer one question at a hook of a running agent. The injection guard rail fails closed: a decision error quarantines the tool result too. | agent | front | what it measures | run | |---|---|---|---| -| `allrecipes` | browser | browser use against the chat model: WebVoyager task Allrecipes--0, a vegetarian lasagna with over 100 reviews, 4.5 stars or more, for 6 | `s1a run allrecipes --slot jev --batch on --headed` | -| `flights` | browser | browser use: wall clock on a Google Flights search | `s1a run flights --slot jev --batch on --profile-out run.json` | -| `desktop` | tool | computer use: clicks toward `--goal` until the window shows `--expect`; the Calculator on macOS or Windows is the example | `s1a run desktop --app Calculator --goal "compute 12 times 7" --expect 84 --execute --clear "All Clear" --slot jev --rethink off --episodes 1` | -| `ticket_router` | tool | correct routes on a seeded batch of 30 labelled tickets, five queues | `s1a run ticket_router --slot jev --rethink off --episodes 1` | -| `alfworld` | tool | success on unseen household tasks in text | `s1a run alfworld --slot jev --rethink on --episodes 12 --stride 11` | -| `game2048` | tool | score and largest tile at a move cap | `s1a run game2048 --slot jev --rethink on --episodes 10` | -| `millionaire` | tool | winnings on a 15-question quiz ladder | `s1a run millionaire --slot jev --rethink off --episodes 5` | -| `blackjack` | tool | payoff per hand (RLCard) | `s1a run blackjack --slot jev --rethink off --episodes 100` | +| `allrecipes` | browser | browser use against the chat model: WebVoyager task Allrecipes--0, a vegetarian lasagna with over 100 reviews, 4.5 stars or more, for 6 | `s1a run allrecipes --model jev --batch on --headed` | +| `flights` | browser | browser use: wall clock on a Google Flights search | `s1a run flights --model jev --batch on --profile-out run.json` | +| `desktop` | tool | computer use: clicks toward `--goal` until the window shows `--expect`; the Calculator on macOS or Windows is the example | `s1a run desktop --app Calculator --goal "compute 12 times 7" --expect 84 --execute --clear "All Clear" --model jev --rethink off --episodes 1` | +| `ticket_router` | tool | correct routes on a seeded batch of 30 labelled tickets, five queues | `s1a run ticket_router --model jev --rethink off --episodes 1` | +| `alfworld` | tool | success on unseen household tasks in text | `s1a run alfworld --model jev --rethink on --episodes 12 --stride 11` | +| `game2048` | tool | score and largest tile at a move cap | `s1a run game2048 --model jev --rethink on --episodes 10` | +| `millionaire` | tool | winnings on a 15-question quiz ladder | `s1a run millionaire --model jev --rethink off --episodes 5` | +| `blackjack` | tool | payoff per hand (RLCard) | `s1a run blackjack --model jev --rethink off --episodes 100` | | `injection_guard` | rail | precision and recall on a labelled injection set | `s1a run injection_guard` | -Every tool agent takes `--slot jev|laya|cua|llm|random|rule`, `--rethink on|off`, `--episodes N`, `--seed S`, +Every tool agent takes `--model jev|laya|cua|llm|random|rule`, `--rethink on|off`, `--episodes N`, `--seed S`, `--max-steps` and `--timeout`, and writes a Harbor-shaped job folder under `evals/results//`. A browser agent -takes `--slot jev|laya|cua|llm` and `--goal`. A rail takes `--slot jev|laya`, the two slots that answer `noul`. -`uv run python -m evals.table evals/results` aggregates every job folder per eval and slot into one table. +takes `--model jev|laya|cua|llm` and `--goal`. A rail takes `--model jev|laya`, the two models that answer `noul`. +`uv run python -m evals.table evals/results` aggregates every job folder per eval and model into one table. Every `run` prints one JSON object on stdout and nothing else there; `s1a-mcp` serves the same agents over stdio with `list_agents`, `run_agent` and `decide`. Flags, exit codes and the job-folder layout: -[architecture.md](architecture.md). The extras each agent needs and the keys: `CONTRIBUTING.md`. The slots and the -models behind them: [architecture.md](architecture.md#slots). +[architecture.md](architecture.md). The extras each agent needs and the keys: `CONTRIBUTING.md`. The `--model` values and the +models behind them: [architecture.md](architecture.md#models). ## Agent-specific flags @@ -39,8 +39,8 @@ models behind them: [architecture.md](architecture.md#slots). ([He et al., 2024](https://arxiv.org/abs/2401.13919), Apache License 2.0, attribution in `NOTICE`), verbatim: find a vegetarian lasagna with more than 100 reviews, a rating of at least 4.5 stars, suitable for 6 people, and answer in text. The run is headed -because the site answers a headless Chromium with a bot wall. `scripts/browser_showcase.sh allrecipes 2` runs both slots -twice with a frame after every browser call and renders the pair GIFs from the median run of each slot; the numbers +because the site answers a headless Chromium with a bot wall. `scripts/browser_showcase.sh allrecipes 2` runs both models +twice with a frame after every browser call and renders the pair GIFs from the median run of each model; the numbers and the replay are in [benchmarks.md](benchmarks.md#browser-use-allrecipes). ## The ticket router diff --git a/docs/architecture.md b/docs/architecture.md index 295a75b..5bfb690 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -23,10 +23,10 @@ sequenceDiagram ``` Every agent is one module under `s1a/agents/` that ends in a frozen `SPEC` (`s1a/spec.py`): the name, -the description, the rules text the model in the slot reads, a budget, and the front-specific pieces. Three fronts share the loop: +the description, the rules text the model reads, a budget, and the front-specific pieces. Three fronts share the loop: - Tool front (`ToolAgentSpec`, `s1a/tool/`): a DeepAgent plays a game through two tools, `observe` and `act`, - with `ToolDecisionModel` in the slot over a decision model; the `random` and `rule` slots put the two baseline + with `ToolDecisionModel` in the slot over a decision model; `random` and `rule` put the two baseline models in the same slot model. Agents: `blackjack`, `game2048`, `millionaire`, `alfworld`, `desktop`, `ticket_router`. The loop: `evals/README.md`. - Browser front (`BrowserAgentSpec`, `s1a/browser/`): `BrowserDecisionModel` fills the slot of openJiuwen's browser subagent. Each browser turn is one `decide_many` over the page's controls, one question per head; the chat @@ -42,9 +42,9 @@ validation; `docs/decision-models.md`), `s1a/decision_models/wire.py` (the HTTP (Harbor-shaped job folders), `s1a/browser/profiler.py` (where a browser run's seconds go). -## Slots +## Models -| `--slot` | model | runs | reads | +| `--model` | who answers | runs | reads | |---|---|---|---| | `jev` | [TypeSafe Jev](https://typesafe.ai) | over HTTP with a key; 350 to 500 ms, $0.042 per million input tokens | up to 32K tokens of state | | `laya` | [Laya](https://huggingface.co/convaiinnovations/laya) (`convaiinnovations/laya`, 0.4B) | in process, `uv sync --extra laya`; no key | a 512 to 1024 token window | @@ -52,7 +52,7 @@ validation; `docs/decision-models.md`), `s1a/decision_models/wire.py` (the HTTP | `random`, `rule` | the tool front's two baselines | in process | the candidates | | `llm` | the chat model | for the comparison columns | the transcript | -`decision_models.build_model(slot)` builds the first five; `llm` is not a decision model. +`decision_models.build_model(model_name)` builds the first five; `llm` is not a decision model. ## Entry points @@ -62,17 +62,17 @@ the first import of both, routes the harness logs to files under `runs/logs` bef MCP stdio protocol only. Every `run` prints one JSON object on stdout: a tool agent's series summary with its `job_dir`, a browser agent's -answer, a rail's evaluation. A browser agent takes `--slot jev|laya|cua|llm`; its policy switches are run-time flags: +answer, a rail's evaluation. A browser agent takes `--model jev|laya|cua|llm`; its policy switches are run-time flags: `--batch on|off`, `--prefetch on|off`, `--goal-values on|off`. `s1a-mcp` serves the same agents to an MCP host over stdio, one Runner for the server's lifetime and one run at a time. `uv run python -m evals.table evals/results` -aggregates every job folder per eval and slot into one table. `scripts/showcase.sh` plays one visual episode per -eval and slot outside the matrix and `python -m evals.replay` renders a pair side by side, with a GIF; see +aggregates every job folder per eval and model into one table. `scripts/showcase.sh` plays one visual episode per +eval and model outside the matrix and `python -m evals.replay` renders a pair side by side, with a GIF; see `evals/README.md`. -Every tool agent, `desktop` included, takes `--slot jev|laya|cua|llm|random|rule`, `--rethink on|off`, +Every tool agent, `desktop` included, takes `--model jev|laya|cua|llm|random|rule`, `--rethink on|off`, `--episodes N`, `--seed S`, `--max-steps`, `--timeout` and `--headed`, and writes a Harbor-shaped job folder under -`evals/results//`. Every browser agent takes `--slot jev|laya|cua|llm` and `--goal`. A rail takes -`--slot jev|laya`, the two slots that answer `noul`. `decide` and `probe` take `--slot jev|laya|cua`. On a browser +`evals/results//`. Every browser agent takes `--model jev|laya|cua|llm` and `--goal`. A rail takes +`--model jev|laya`, the two models that answer `noul`. `decide` and `probe` take `--model jev|laya|cua`. On a browser agent `laya` needs `LAYA_MAX_LEN` raised to the page's size; `cua` reads a 256-byte context (header, goal, state, then rules) and 96 bytes per option, a baseline on any page. Exit codes: 0 for a finished run, including one whose JSON has `ok: false`; 1 for a run, key, model or file error, one line on stderr; 2 for a usage error (an unknown @@ -81,11 +81,11 @@ agent, a malformed `--state`, `--option`, `@file` or cases file, one line on std `confidence` and `ms`. The harness logs go to files under `runs/logs`. -## Why the decision-model slot is faster +## Why a decision model is faster -The decision-model slot reads `env.observe()` (or the page probe) each turn and answers in one request of 350 to 500 ms +A decision model reads `env.observe()` (or the page probe) each turn and answers in one request of 350 to 500 ms whose cost does not grow with the episode. The chat model reads the tool-result transcript, which grows every -turn, and writes the tool call as text. Both slots see the same observation and the same rules. The model in the slot is the +turn, and writes the tool call as text. Both models see the same observation and the same rules. The model in the slot is the one difference. ## Dependencies diff --git a/docs/benchmarks.md b/docs/benchmarks.md index dd3f68e..a804c05 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,7 +1,7 @@ # Benchmarks: Jev against the chat model in the same agent Every number here compares a System 1 decision model against a chat model in the same agent, on the same tools and -the same seeds. The model in the slot is the only difference between the two columns. +the same seeds. The model is the only difference between the two columns. ## Six agents, one episode each @@ -27,14 +27,14 @@ that shows the expected result. The browser row is in the next section. The `allrecipes` agent runs the first Allrecipes task of the [WebVoyager](https://github.com/MinorJerry/WebVoyager) task set verbatim (`Allrecipes--0` in `data/WebVoyager_data.jsonl`; [He et al., 2024](https://arxiv.org/abs/2401.13919); Apache License 2.0, attribution in `NOTICE`): a vegetarian lasagna with more than 100 reviews, a rating of at least -4.5 stars, suitable for 6 people, answered in text. Both slots ran headed on +4.5 stars, suitable for 6 people, answered in text. Both models ran headed on 2026-09-22 (the site answers a headless Chromium with a bot wall) on the Playwright MCP without its settle sleeps, with a frame after every browser call from the run's own MCP session: `scripts/browser_showcase.sh allrecipes 2`. The chat model on both sides is Claude Fable 5.1 through OpenRouter, at $10 per million input tokens, $50 per million output tokens and $0.25 per million cached input tokens; Jev's decisions go to TypeSafe at $0.042 per -million input tokens. The Jev slot pays the chat model for one typed value, the search text, and for the answer. +million input tokens. The Jev run pays the chat model for one typed value, the search text, and for the answer. -| run | slot | wall s | decisions | chat calls | chat tokens in / cached / out | Jev tokens | cost | answer | +| run | model | wall s | decisions | chat calls | chat tokens in / cached / out | Jev tokens | cost | answer | |---|---|---|---|---|---|---|---|---| | 0 | jev | 35.7 | 5 | 4 | 7,946 / 1,210 / 454 | 23,654 | $0.0914 | Easy Vegetarian Spinach Lasagna: 4.6 stars, 117 ratings, serves 6 | | 1 | jev | 40.4 | 4 | 4 | 7,335 / 0 / 440 | 18,785 | $0.0961 | the same recipe | @@ -46,7 +46,7 @@ million input tokens. The Jev slot pays the chat model for one typed value, the Every answer meets the task's three conditions, checked by hand on the recipe page: 4.6 stars, 117 ratings (85 of them written reviews; the WebVoyager judge of the September 19 batch accepted the ratings count as reviews), 6 servings. The README row and the replay are Jev's run 0 against the chat model's run 2, the median run of each -slot by wall clock; the chat model's cost is 16.4 times Jev's. The records of those two runs are under `results/allrecipes/`. +model by wall clock; the chat model's cost is 16.4 times Jev's. The records of those two runs are under `results/allrecipes/`. Two more pairs ran the same afternoon without records in that folder. The first, before the probe dropped escaped markup from control labels: Jev answered in 31.9 s of process time for $0.102 with the same recipe, after two clicks @@ -56,7 +56,7 @@ overwrote: Jev 40.3 s, 5 decisions and $0.096; the chat model 104.9 s, 8 decisio lasagna. `wall s` is the task's own clock, written to `answer.json`: the browser's start, the navigation, every decision and -the final answer. Of the Jev slot's cost, the decisions themselves are a tenth of a cent; the rest is the chat +the final answer. Of the Jev run's cost, the decisions themselves are a tenth of a cent; the rest is the chat model's typed value and answer. Jev's run 2 cost a fifth of run 1 because it read 7,295 of its 7,335 prompt tokens from the cache. The frames add one screenshot per browser call on both sides. Each run begins with a chat-model call by the harness that probes the model's image support. OpenRouter refuses that call for this model and @@ -69,7 +69,7 @@ the harness continues without it. Both sides pay it. Over 150 moves of 2048 the cost gap widens with the transcript. Seed 0: Jev scored 1,104 in 244 s for $0.003; the chat model scored 1,188 in 363 s for $0.31. -![2048, seed 0, 150 moves. Left: Jev in the slot. Right: the chat model in the same slot.](results/2048/showcase/replay.gif) +![2048, seed 0, 150 moves. Left: Jev. Right: the chat model.](results/2048/showcase/replay.gif) The same showcase on ALFWorld, Blackjack and Millionaire is under `results//showcase/replay.gif`, written by `scripts/showcase.sh`. @@ -83,7 +83,7 @@ The rail scores 20 of 20 on its labelled set at a median of 464 ms (`s1a run inj The protocol in `evals/README.md` quotes nothing under ten episodes and asks for 500 Blackjack hands. The series of 2026-09-19, from job folders on the author's machine; the chat model of the `llm` rows is not recorded in them: -| eval | slot | N | score, 95 % CI | s per episode | steps | $ per episode | +| eval | model | N | score, 95 % CI | s per episode | steps | $ per episode | |---|---|---|---|---|---|---| | Blackjack | jev | 100 | -0.06 [-0.25, 0.13] | 0.6 | 1.5 | 0.0000 | | Blackjack | llm | 100 | -0.06 [-0.25, 0.13] | 3.0 | 1.5 | 0.0006 | @@ -93,9 +93,9 @@ The protocol in `evals/README.md` quotes nothing under ten episodes and asks for | ALFWorld text | oracle plan | 12 | 0.917 [0.75, 1.00] | 0.4 | 19.3 | 0 | | 2048 | jev | 5 | 1115 [924, 1238] | 168.9 | 146.6 | 0.0032 | -On Blackjack the three slots play the identical basic strategy over 100 hands. Jev takes a fifth of the chat +On Blackjack the three models play the identical basic strategy over 100 hands. Jev takes a fifth of the chat model's time per hand. On ALFWorld the chat model wins two more games of twelve. It spends 16 times the dollars. -`uv run python -m evals.table evals/results` prints this table for any run. The `laya` and `cua` slots have no +`uv run python -m evals.table evals/results` prints this table for any run. `laya` and `cua` have no numbers yet. ## Google Flights: four drivers on one clock @@ -110,7 +110,7 @@ Goal for every arm: open Google Flights, find one-way flights from Zurich to Lon - **C', S1A on Playwright without the settle sleeps**: arm C with one change in the MCP server: `waitForCompletion` no longer sleeps 500 ms before and after waiting for in-flight requests. `scripts/pw_mcp_nosettle.sh` prepares that copy; it is a patched npm package. - **C'', S1A on Playwright, no settle sleeps, batched actions**: arm C' with `--batch on`: each step is one `browser_run_code_unsafe` call that performs the action and returns the next probe, one transport round trip per step where C' uses two. The runtime's target validation is skipped on that path; the batched code re-stamps and re-finds the target by role and label when a re-render dropped the stamp. -Per run, S1A arms record the profiler JSON (`s1a run flights --slot jev --batch on --profile-out run.json`); arm A records jev-ultrafast's `state.json`. `scripts/summarize_runs.py` prints the table from those files. The S1A records live under `docs/results/flights/`. The arm A records stay out of the tree because each holds a page screenshot. The shipped `flights` agent computes its date as the first Sunday at least 28 days after the run day; the runs below used September 20, 2026. The records' `visible_flights` and screenshots show prices in yen because the runs were made from Japan; Google picks the currency from the run's location. +Per run, S1A arms record the profiler JSON (`s1a run flights --model jev --batch on --profile-out run.json`); arm A records jev-ultrafast's `state.json`. `scripts/summarize_runs.py` prints the table from those files. The S1A records live under `docs/results/flights/`. The arm A records stay out of the tree because each holds a page screenshot. The shipped `flights` agent computes its date as the first Sunday at least 28 days after the run day; the runs below used September 20, 2026. The records' `visible_flights` and screenshots show prices in yen because the runs were made from Japan; Google picks the currency from the run's location. ### Results @@ -231,7 +231,7 @@ cp -R "$CACHE" /tmp/pw-mcp-nosettle # In /tmp/pw-mcp-nosettle/node_modules/playwright-core/lib/coreBundle.js, delete the two # `await tab2.waitForTimeout(500);` lines inside `async function waitForCompletion`. PLAYWRIGHT_MCP_COMMAND=node PLAYWRIGHT_MCP_ARGS=/tmp/pw-mcp-nosettle/node_modules/@playwright/mcp/cli.js \ - s1a run flights --slot jev --batch off --profile-out run.json # TYPESAFE_API_KEY unset: the OpenRouter proxy + s1a run flights --model jev --batch off --profile-out run.json # TYPESAFE_API_KEY unset: the OpenRouter proxy ``` ### Six arms, one table diff --git a/docs/browser-front.md b/docs/browser-front.md index 7b67345..026ff92 100644 --- a/docs/browser-front.md +++ b/docs/browser-front.md @@ -21,8 +21,8 @@ browser-use/jev-ultrafast (MIT), whose observe-decide-act tick this policy follo 1. A decision model fills the `Model` slot. `BrowserDecisionModel(Model)` answers a browser turn (the tool list holds `browser_click`) with exactly one `browser_*` tool call and forwards every other turn (summaries, typed values) to the wrapped chat model. DeepAgent, rails, checkpoints and the permission engine are untouched. - The model behind it is a `DecisionModel`: TypeSafe Jev over HTTP with `--slot jev`, Laya in - process with `--slot laya`; the policy is the same. `action_space.py` builds the `Observation` and the + The model behind it is a `DecisionModel`: TypeSafe Jev over HTTP with `--model jev`, Laya in + process with `--model laya`; the policy is the same. `action_space.py` builds the `Observation` and the typed questions of a tick (`build_observation`, `build_questions`) and reads the answer back onto a candidate (`interpret`). 2. The policy has its own in-page probe. `probe_js.py` waits for the page to settle and describes every diff --git a/docs/decision-models.md b/docs/decision-models.md index f241dbb..915cd38 100644 --- a/docs/decision-models.md +++ b/docs/decision-models.md @@ -1,4 +1,4 @@ -# The decision-model layer: one interface between every front and the model in the slot +# The decision-model layer: one interface between every front and the decision model Code: `s1a/decision_models/`. Tests: `tests/test_decision_models_*.py`, the shared contract in `tests/decision_model_contract.py`. The wire itself (the async HTTP client, the `typesafe` and `openrouter` backends) @@ -7,10 +7,10 @@ stays in `s1a/decision_models/wire.py` and reads no answer. Every front that asks "which one" (the tool loop, the browser policy, the rails, `decide`, `probe`, the MCP server) talks only to `DecisionModel`; the wire client is private to `s1a/decision_models/`. A decision model is a classifier over options the caller enumerates. It reads a state and returns a distribution over the offered keys. Hugging Face writes "System 1 decision model" and -TypeSafe "System One model". This repository uses the terms interchangeably. The backend is a slot: `jev` (TypeSafe Jev over HTTP), +TypeSafe "System One model". This repository uses the terms interchangeably. `--model` picks the backend: `jev` (TypeSafe Jev over HTTP), `laya` (in process, behind `uv sync --extra laya`), `cua` (Cua-S1 Nano in process, behind `uv sync --extra cua`), `random` and `rule` (the tool front's baselines). -`build_model(slot, seed=, rule=)` builds one from the environment; `llm` names the chat model, which `build_model` does not build. +`build_model(model_name, seed=, rule=)` builds one from the environment; `llm` names the chat model, which `build_model` does not build. ## The interface @@ -40,7 +40,7 @@ shorthands; `warm()` and `close()` open and release the backend. ## Backends -| slot | class | `name` | notes | +| `--model` | class | `name` | notes | |---|---|---|---| | `jev` | `JevModel(transport)` | `jev` | the request body every front sent before the layer existed, byte for byte; `from_env` picks TypeSafe or the OpenRouter proxy | | `laya` | `LayaModel(agent, model=)` | `laya` | one forward pass per call on a thread; `MODEL_SERVICE_CONFIG_ERROR` when `input_tokens` fills the window (Laya cuts the state silently; `LAYA_MAX_LEN`, `LAYA_HEAD_MAX_LEN` widen it); `ValueError` and `RuntimeError` from the library become `MODEL_CALL_FAILED` | @@ -50,7 +50,7 @@ shorthands; `warm()` and `close()` open and release the backend. `name` lands in every tick's `source` and in `Episode.policy`; the eval table's columns take their labels from it. -TypeSafe Jev fills the `jev` slot. One request holds a `state` and one or more questions over options the caller +TypeSafe Jev answers `--model jev`. One request holds a `state` and one or more questions over options the caller enumerates; the answer holds one option per question, a probability per option and a confidence, from one forward pass, with no free text. Three heads: `choice` picks one key among the options, `noul` gives the probability that a statement holds, `score` places the state on an ordered rubric. Input is capped at 32K tokens; the endpoint is @@ -83,8 +83,8 @@ real; `bodies` records every request). `ScriptedModel` fakes the interface for f `supports_images`, `question_types`, `deterministic`; implement `model` and `_decide`, which translates the questions and returns a `Reply` whose `answers` are the backend's own dicts; `decide_many` validates them into a `Decision`. Keep any heavy import inside `from_env()`. -2. A `case` in `factory.build_model` and the slot name in `DECISION_MODEL_SLOTS`, `tool/loop.py::SLOTS`, - `browser/browse.py::BROWSER_SLOTS`, `rails.RAIL_SLOTS` and `cli.DECIDE_SLOTS`. +2. A `case` in `factory.build_model` and the name in `DECISION_MODEL_NAMES`, `tool/loop.py::MODEL_NAMES`, + `browser/browse.py::BROWSER_MODEL_NAMES`, `rails.RAIL_MODEL_NAMES` and `cli.DECIDE_MODEL_NAMES`. 3. `tests/test_decision_models_.py` with `TestContract(DecisionModelContract, IsolatedAsyncioTestCase)` plus the backend's mapping tests; a fake for its SDK lives in that file. 4. An optional extra in `pyproject.toml` and an env block in `.env.example` when it needs a dependency. @@ -92,4 +92,4 @@ real; `bodies` records every request). `ScriptedModel` fakes the interface for f Laya is text only and reads a 512 to 1024 token window; it fits the tool front first. The browser front's element tables are wider than that window. The window check sums `input_tokens` over the request's questions. On the browser front (two to four questions per tick) only a cut on every head raises the config error above; a cut on -one head goes unseen. `--slot laya` on a browser agent needs `LAYA_MAX_LEN` raised to the page's size. +one head goes unseen. `--model laya` on a browser agent needs `LAYA_MAX_LEN` raised to the page's size. diff --git a/docs/results/allrecipes/jev/answer.json b/docs/results/allrecipes/jev/answer.json index db044c2..000ba12 100644 --- a/docs/results/allrecipes/jev/answer.json +++ b/docs/results/allrecipes/jev/answer.json @@ -67,5 +67,5 @@ "answer": "The page shows \"Easy Vegetarian Spinach Lasagna\" (by MOTTSBELA), rated 4.6 stars with 117 ratings but only 85 written reviews, serving 6 (prep 20 min, cook 1 hr, total 1 hr 25 min; ingredients: 10 oz fresh spinach, 9 lasagna noodles, 8 oz ricotta, 1 cup mozzarella, 1 egg, pinch nutmeg, pinch basil, 1 cup pasta sauce, ¼ cup Parmesan, salt/pepper, cooking spray; layer noodles, spinach-ricotta mix, and sauce in a 1½-qt dish, top with mozzarella and Parmesan, bake covered at 400°F for 25 min then uncovered ~25 min more)." }, "agent": "allrecipes", - "slot": "jev" + "model": "jev" } \ No newline at end of file diff --git a/docs/results/allrecipes/llm/answer.json b/docs/results/allrecipes/llm/answer.json index 4e97071..cc13611 100644 --- a/docs/results/allrecipes/llm/answer.json +++ b/docs/results/allrecipes/llm/answer.json @@ -101,5 +101,5 @@ "chat_temperature": 0.0 }, "agent": "allrecipes", - "slot": "llm" + "model": "llm" } \ No newline at end of file diff --git a/docs/roadmap.md b/docs/roadmap.md index c0fcc82..ae1fdcf 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -97,7 +97,7 @@ free-text negotiation decide the score). ## Order of work -1. The slot matrix on the built evals, with cost and decision counts (in progress). +1. The model matrix on the built evals, with cost and decision counts (in progress). 2. The tool-call guardrail next to the shipped injection guard, as one "Jev security rails" feature, with the labelled sets. 3. Severity and escalation rail, measured on stored trajectories. diff --git a/docs/skills.md b/docs/skills.md index f95b5e0..3056b6c 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -40,15 +40,15 @@ uv run s1a decide \ One JSON object comes back in about 400 ms: `choice`, a probability per queue, `confidence` and `ms`. The shipped `ticket_router` agent uses the same queues and rules: it routes a seeded batch of 30 labelled tickets and scores the -correct routes. The first row of the README's table is one such batch on each slot: +correct routes. The first row of the README's table is one such batch with each model: ```bash -uv run s1a run ticket_router --slot jev --rethink off --episodes 1 -uv run s1a run ticket_router --slot llm --rethink off --episodes 1 +uv run s1a run ticket_router --model jev --rethink off --episodes 1 +uv run s1a run ticket_router --model llm --rethink off --episodes 1 ``` A page task goes the same way. The prompt names the site, the values to enter and the stop condition; the skill -runs `s1a run flights --slot jev --goal "..."` and reads the answer from `final` in the JSON. +runs `s1a run flights --model jev --goal "..."` and reads the answer from `final` in the JSON. ### What to delegate @@ -74,7 +74,7 @@ codex mcp add s1a -- uv run --project /path/to/system1-agents s1a-mcp `s1a-mcp` serves the same agents over stdio as three tools. `list_agents()` returns every agent with its front, its description and the flags `run_agent` accepts for it; an agent whose optional dependency is missing is listed as unavailable with the error. `run_agent(name, flags)` runs one agent with the flags of `s1a run ` and returns -its JSON object. `decide(state, options, rules)` answers one choice question on the `jev` slot: the chosen key, a +its JSON object. `decide(state, options, rules)` answers one choice question with `jev`: the chosen key, a probability per option, a confidence and the latency in ms. ## Build a System 1 agent @@ -95,5 +95,5 @@ It produces the module, its test and a row in the agents table, and stops at the controls, a rail for one question at a hook of a running agent. 4. Scaffold from the front's template under `s1a/agents/_templates/`, with the state-design rules from the skill's references. -5. Verify one rung at a time: the offline test, then `--slot random`, `rule`, `jev` and `llm` on the same seeds, then +5. Verify one rung at a time: the offline test, then `--model random`, `rule`, `jev` and `llm` on the same seeds, then the results table and the full suite. diff --git a/docs/why.md b/docs/why.md index 637aa94..570dda4 100644 --- a/docs/why.md +++ b/docs/why.md @@ -50,7 +50,7 @@ neither. driver (browser-use sidecar or Playwright MCP) is the driver. A driver has eyes and hands only. The decision belongs above it. (The agtai/agent-core fork's F_04 note, "Rejected") 3. One loop, swap the brain. Every eval runs the same DeepAgent with the same two tools and the same rules - text; `--slot jev|llm|random|rule|laya|cua` changes only the model in the slot. Score, seconds, steps, decisions and dollars + text; `--model jev|llm|random|rule|laya|cua` changes only the model. Score, seconds, steps, decisions and dollars are compared on the same seeds. (`evals/README.md`) 4. System 1 for Jev, System 2 for the chat model. Fast recognition (which control, which move, is this text an instruction, is this call safe) goes to Jev. Planning, arithmetic, constraint solving, typed values and the @@ -71,7 +71,7 @@ neither. 9. Measure against a fair reference on one clock. Timings are quoted only from runs where every arm used the same Chrome, the same decisions backend and the same chat model, back to back. Day-to-day drift is attributed by component before any claim is made. (`docs/benchmarks.md`) -10. Slots upstream, fronts here. openJiuwen receives generic seams (the `DecisionPolicyModel` Protocol and +10. Seams upstream, fronts here. openJiuwen receives generic seams (the `DecisionPolicyModel` Protocol and two runtime hooks, about 100 lines). Every Jev-specific front lives in this repository. (`docs/roadmap.md`, "Upstream asks") @@ -104,10 +104,10 @@ plugin delegates through the local Codex CLI and app server wrapped in a subagen The plan follows that norm. -1. CLI first. `s1a run flights --slot jev --goal ""` runs the browser subagent with Jev in the slot and +1. CLI first. `s1a run flights --model jev --goal ""` runs the browser subagent with Jev as its model and prints the answer as one JSON object on stdout; the harness logs go under `runs/logs`. `s1a decide --state @file --option a=... --option b=... --rules "..."` exposes the `choice` primitive. - `s1a run --slot jev --episodes N` runs a registered eval. A process start costs about 1.3 s (import of + `s1a run --model jev --episodes N` runs a registered eval. A process start costs about 1.3 s (import of the openjiuwen browser stack). That is small next to a browse task. For `decide` it is three times the decision itself. 2. One skill, three hosts. `skills/s1a/SKILL.md` in the agentskills.io layout states when to call S1A diff --git a/evals/README.md b/evals/README.md index 07f43b3..256b11d 100644 --- a/evals/README.md +++ b/evals/README.md @@ -6,13 +6,13 @@ folder per episode). `uv run python -m evals.table evals/results` aggregates the | eval | measures | run | |---|---|---| -| Blackjack (RLCard) | payoff per hand | `s1a run blackjack --slot jev --rethink off --episodes 100` | -| 2048 (the MIT game, self-hosted) | score and largest tile at a move cap | `s1a run game2048 --slot jev --rethink on --episodes 10` | -| Millionaire (self-hosted quiz, Open Trivia DB) | winnings; the 50:50 lifeline is a candidate the model may pick; six ladders: `--seed` plus `--episodes` stays at or below 6 | `s1a run millionaire --slot jev --rethink off --episodes 5` | -| ALFWorld text (TextWorld) | success on unseen games | `s1a run alfworld --slot jev --rethink on --episodes 12 --stride 11` | -| Google Flights timing | wall clock against browser-use/jev-ultrafast | `s1a run flights --slot jev --batch on`; numbers in `docs/benchmarks.md` | -| Desktop (Cua Driver) | clicks toward `--goal` until the window shows `--expect` | `s1a run desktop --app Calculator --goal "compute 12 times 7" --expect 84 --execute --slot jev --rethink off --episodes 1` | -| Ticket router (30 local labelled tickets) | correct routes to five queues | `s1a run ticket_router --slot jev --rethink off --episodes 1` | +| Blackjack (RLCard) | payoff per hand | `s1a run blackjack --model jev --rethink off --episodes 100` | +| 2048 (the MIT game, self-hosted) | score and largest tile at a move cap | `s1a run game2048 --model jev --rethink on --episodes 10` | +| Millionaire (self-hosted quiz, Open Trivia DB) | winnings; the 50:50 lifeline is a candidate the model may pick; six ladders: `--seed` plus `--episodes` stays at or below 6 | `s1a run millionaire --model jev --rethink off --episodes 5` | +| ALFWorld text (TextWorld) | success on unseen games | `s1a run alfworld --model jev --rethink on --episodes 12 --stride 11` | +| Google Flights timing | wall clock against browser-use/jev-ultrafast | `s1a run flights --model jev --batch on`; numbers in `docs/benchmarks.md` | +| Desktop (Cua Driver) | clicks toward `--goal` until the window shows `--expect` | `s1a run desktop --app Calculator --goal "compute 12 times 7" --expect 84 --execute --model jev --rethink off --episodes 1` | +| Ticket router (30 local labelled tickets) | correct routes to five queues | `s1a run ticket_router --model jev --rethink off --episodes 1` | | Injection guard (rail) | precision and recall on a labelled set | `s1a run injection_guard` | ## The loop @@ -20,7 +20,7 @@ folder per episode). `uv run python -m evals.table evals/results` aggregates the `s1a/tool/loop.py` builds one DeepAgent per episode through `create_deep_agent`, with two tools per environment. `observe` returns `{state, candidates, done, score}`; `act(key)` plays one candidate key and returns the same shape. The model slot holds `ToolDecisionModel` (`s1a/tool/models.py`) over one of five decision models, or the -chat model (`--slot`): +chat model (`--model`): - `jev`: the model reads the environment, asks Jev one choice question, and answers with one `act` call. - `laya`: the same, with Laya deciding in process (`uv sync --extra laya`); its tokens are free and unpriced. @@ -41,7 +41,7 @@ identical call) stays off for these agents. Every episode records its ticks (key, confidence, probabilities, latency, tokens, whether a plan was in the state) and its rethink events in `agent/episode.json`; the count of chat-model calls and their token sums go to -`result.json`. For the `llm` slot the decisions are the chat calls that produced an `act`; a tick whose key was +`result.json`. For `llm` the decisions are the chat calls that produced an `act`; a tick whose key was not offered is kept with `accepted: false`, counted as an `invalid_key`, and skipped by the replay. Its iteration cap is twice the game's budget, since the chat model spends turns on unknown keys. A run that reaches the iteration cap has `result_type: error` in `extra`; its score still counts. An episode that @@ -53,7 +53,7 @@ the score statistics over the `scored` episodes only, and `evals.table` skips tr `uv sync`, plus `--extra blackjack` and `--extra alfworld` for those games (the README lists every extra), then a `.env` with `TYPESAFE_API_KEY` or `OPENROUTER_API_KEY` for Jev. -The `llm` slot, the rethink planner and the browser agent need the chat model: `OPENAI_API_KEY` (or `LLM_API_KEY`), +`llm`, the rethink planner and the browser agent need the chat model: `OPENAI_API_KEY` (or `LLM_API_KEY`), `OPENAI_BASE_URL` (or `LLM_BASE_URL`) and `MODEL_NAME`. `MODEL_PROVIDER=anthropic` talks Anthropic's own protocol, direct (`OPENAI_BASE_URL=https://api.anthropic.com`, `MODEL_NAME=claude-fable-5-1`; an org-level key also needs `ANTHROPIC_WORKSPACE_ID`) or through OpenRouter's `/v1/messages` (`MODEL_NAME=anthropic/claude-fable-5.1`). The browser agents launch a headless @@ -63,18 +63,18 @@ of alfworld/alfworld. The DeepAgent's workspace files land under `runs/evals/`. ## Protocol -Every slot plays seeds `S` to `S + N - 1` (`--seed S --episodes N`) and meets the same deals and games. +Every model plays seeds `S` to `S + N - 1` (`--seed S --episodes N`) and meets the same deals and games. `summary.json` holds the mean score with a 95 % bootstrap interval, wins and losses, mean steps, the median decision latency and the rethink count. Quote nothing below ten episodes; Blackjack wants 500 hands. The four measurements: `jev` against `llm` on the same -seeds (same loop, swap the brain); `jev` with `--rethink on` against `off`; the `random` slot's wall clock per +seeds (same loop, swap the brain); `jev` with `--rethink on` against `off`; the `random` model's wall clock per act against a bare loop (loop overhead); and, later, Jev's top probability against the ALFWorld expert plan. `summary.json` also holds decisions, chat calls, tokens (`chat_input_tokens`, `chat_output_tokens`, `chat_cache_tokens`) and `cost_usd` (Jev at $0.042 per M input tokens; the chat model at OpenRouter's catalogue price for `MODEL_NAME`, with cached input at the catalogue's cache-read rate; or `CHAT_USD_PER_M_INPUT`, -`CHAT_USD_PER_M_OUTPUT` and, optionally, `CHAT_USD_PER_M_CACHED_INPUT`). `python -m evals.table evals/results` prints one row per eval and slot over every +`CHAT_USD_PER_M_OUTPUT` and, optionally, `CHAT_USD_PER_M_CACHED_INPUT`). `python -m evals.table evals/results` prints one row per eval and model over every job folder. ALFWorld's game files sort by task type; `--stride 11` from offset 0 takes twelve games across -the six types. Every slot plays the same tile draws because 2048 seeds the page's `Math.random`. +the six types. Every model plays the same tile draws because 2048 seeds the page's `Math.random`. ## Showcase runs and replays @@ -84,11 +84,11 @@ the state and the candidate keys before each decision, and the final state. That ``--showcase`` on any game writes the job under ``evals/showcase/`` (``evals.table`` never reads that tree) and, for 2048 and Millionaire, one PNG of the page per move into ``agent/frames/`` through the runtime, headless or headed. -``scripts/showcase.sh [SEED] [EVALS...]`` plays one episode per eval and slot on the same seed, renders each pair +``scripts/showcase.sh [SEED] [EVALS...]`` plays one episode per eval and model on the same seed, renders each pair under ``evals/showcase/replays//`` and copies the GIF to ``docs/results//showcase/``. ``python -m evals.replay [] --out DIR [--gif]`` (the ``report`` extra: Pillow and Playwright) writes a -page with the two slots side by side on the episode's own clock: the hands for Blackjack, the board for 2048, the +page with the two models side by side on the episode's own clock: the hands for Blackjack, the board for 2048, the question card for Millionaire, the transcript for ALFWorld, and under each the chosen key, Jev's probabilities over the candidates, the latency and the cumulative seconds. ``--gif`` screenshots the page per tick of episode time (``--mode time --speed 4``) or per step (``--mode step``). ``--from-frames DIR [DIR]`` stitches one or two folders of @@ -100,8 +100,8 @@ request and response times in ``calls.jsonl`` beside them. The policy behaves as second browser client is involved. The frames hold the page and nothing else. ``python -m evals.replay --out DIR --gif --strip`` takes two runs' logs folders (``answer.json`` inside, the frames under ``frames/``) as trials: the page draws the frame at the clock with the tick's target probabilities under it, and ``--strip`` writes -the frames side by side under a header band. ``scripts/browser_showcase.sh [RUNS]`` runs both slots headed -with the frames on and renders both GIFs from the median run of each slot. +the frames side by side under a header band. ``scripts/browser_showcase.sh [RUNS]`` runs both models headed +with the frames on and renders both GIFs from the median run of each model. ALFWorld's scenes render through AI2-THOR: ``evals/replay/thor_replay.py `` replays the trial's commands in ``AlfredThorEnv`` and writes a frame per step; the page shows it above the transcript. It needs the ``alfworld-visual`` @@ -114,7 +114,7 @@ it for both ALFWorld trials before the page; without the extra the GIF holds the Smoke-tested through the agent, one to three episodes each, Jev on the direct TypeSafe backend: Blackjack (rule and jev), 2048 (jev with rethink on: 42 acts, 6 repeat blocks, 1.1 s per act, Jev median 382 ms), Millionaire (jev: 32,000 after 12 answers), ALFWorld (both won; the oracle plan in 12 steps, jev in 5). -Series of 2026-09-19 (Blackjack N=100 per slot, ALFWorld N=12 per slot, 2048 N=5) are tabulated in +Series of 2026-09-19 (Blackjack N=100 per model, ALFWorld N=12 per model, 2048 N=5) are tabulated in `docs/benchmarks.md`; the 500-hand Blackjack series the protocol asks for has not been run. Known gaps: the semantic-stall plan has not fired in a live run yet (2048 kept scoring). On Google Flights--1 @@ -127,8 +127,8 @@ Jev paged the date picker back and forth and ended BLOCKED after 18 requests (th three times is a legal 2048 line. Exact repeats are `RethinkRail`'s first layer. 3. The rail and the slot model share one in-memory `EvalState` per episode. Session state was rejected. 4. Millionaire's 50:50 is a candidate key the model may pick when it is available, in place of a threshold wrapper. -5. A `rule` slot keeps the hand-written baselines in the same loop. -6. The bare loop is gone; the loop-overhead number comes from the `random` slot against the last bare-loop run +5. `--model rule` keeps the hand-written baselines in the same loop. +6. The bare loop is gone; the loop-overhead number comes from `random` against the last bare-loop run (2048 random, 0.87 s per act). 7. Stall thresholds per game: 2048 six acts, ALFWorld eight, Blackjack and Millionaire none. 8. One slot model over one decision-model interface: `ToolDecisionModel` holds a decision model, and its `name` is diff --git a/evals/replay/cast.py b/evals/replay/cast.py index c067b68..a33c26c 100644 --- a/evals/replay/cast.py +++ b/evals/replay/cast.py @@ -4,14 +4,14 @@ PLAYWRIGHT_MCP_COMMAND= \\ PLAYWRIGHT_MCP_ARGS="-m evals.replay.cast --frames runs/cast/flights-jev -- npx -y @playwright/mcp@0.0.78" \\ - s1a run flights --slot jev --headed + s1a run flights --model jev --headed jiuwen launches the MCP server from those two variables and appends its own flags (``--isolated``, ``--headless``) to the args, which land after ``--`` and reach the real server. The proxy forwards every JSON-RPC line both ways; after each ``tools/call`` of a ``browser_*`` tool it sends one ``browser_take_screenshot`` of its own, keeps that response for itself and saves the PNG as ``t--.png``; ``calls.jsonl`` beside the frames holds each call's tool name and its request and response times on the same clock. The run's own client is the only client of the browser, so the policy -behaves as it does without frames, one screenshot per step later. Two such folders, one per slot, go side by side +behaves as it does without frames, one screenshot per step later. Two such folders, one per model, go side by side on the wall clock with ``python -m evals.replay --from-frames``. """ diff --git a/evals/replay/gif.py b/evals/replay/gif.py index 58f89d3..534803e 100644 --- a/evals/replay/gif.py +++ b/evals/replay/gif.py @@ -21,7 +21,7 @@ STEP_DURATION_MS = 700 MIN_FRAME_MS = 20 HEADER_HEIGHT = 84 # the strip's header band at a column width of 800 px; scales with the column -SLOT_COLORS = {"jev": "#0f766e", "llm": "#b45309"} # the replay page's badge colours +MODEL_COLORS = {"jev": "#0f766e", "llm": "#b45309"} # the replay page's badge colours INK, MUTED, LINE, DONE, PANEL, OTHER = "#1c1f26", "#6b7280", "#e3e6eb", "#2563eb", "#ffffff", "#6b7280" @@ -124,8 +124,8 @@ def strip_columns(trials: list[Trial]) -> list[Column]: columns.append( Column( frames=trial.frames, - badge="LLM" if trial.slot == "llm" else f"SYSTEM 1 · {trial.slot.upper()}", - color=SLOT_COLORS.get(trial.slot, OTHER), + badge="LLM" if trial.model == "llm" else f"SYSTEM 1 · {trial.model.upper()}", + color=MODEL_COLORS.get(trial.model, OTHER), facts=f"{trial.elapsed_s:.1f} s · {trial.steps} steps · {len(trial.decisions)} decisions · {cost}", total_ms=round(trial.elapsed_s * 1000), ) diff --git a/evals/replay/page.py b/evals/replay/page.py index 2f2772e..0298c12 100644 --- a/evals/replay/page.py +++ b/evals/replay/page.py @@ -18,7 +18,7 @@ def trial_data(trial: Trial, frames: list[str]) -> dict[str, Any]: return { - "slot": trial.slot, + "model": trial.model, "seed": trial.seed, "score": trial.score, "elapsed_s": trial.elapsed_s, @@ -38,12 +38,12 @@ def trial_data(trial: Trial, frames: list[str]) -> dict[str, Any]: def copy_frames(trials: list[Trial], out_dir: Path) -> list[list[str]]: - """Each trial's frames copied under ``out_dir/frames-/``; the page references them by that relative path.""" + """Each trial's frames copied under ``out_dir/frames-/``; the page references them by that relative path.""" copied: list[list[str]] = [] seen: dict[str, int] = {} for trial in trials: - seen[trial.slot] = seen.get(trial.slot, 0) + 1 - folder = f"frames-{trial.slot}" + (f"-{seen[trial.slot]}" if seen[trial.slot] > 1 else "") + seen[trial.model] = seen.get(trial.model, 0) + 1 + folder = f"frames-{trial.model}" + (f"-{seen[trial.model]}" if seen[trial.model] > 1 else "") paths: list[str] = [] if trial.frames: target = out_dir / folder @@ -56,11 +56,11 @@ def copy_frames(trials: list[Trial], out_dir: Path) -> list[list[str]]: return copied -SLOT_ORDER = {"jev": 0, "llm": 1} # Jev on the left, the chat model on the right, everything else after +MODEL_ORDER = {"jev": 0, "llm": 1} # Jev on the left, the chat model on the right, everything else after def ordered(trials: list[Trial]) -> list[Trial]: - return sorted(trials, key=lambda trial: SLOT_ORDER.get(trial.slot, len(SLOT_ORDER))) + return sorted(trials, key=lambda trial: MODEL_ORDER.get(trial.model, len(MODEL_ORDER))) def render_page(trials: list[Trial], *, out_dir: Path) -> str: @@ -73,7 +73,7 @@ def render_page(trials: list[Trial], *, out_dir: Path) -> str: "trials": [trial_data(trial, paths) for trial, paths in zip(trials, frames)], } payload = json.dumps(data, ensure_ascii=False).replace("<", "\\u003c") - title = f"{data['eval']}: {' vs '.join(t['slot'] for t in data['trials'])}" + title = f"{data['eval']}: {' vs '.join(t['model'] for t in data['trials'])}" return TEMPLATE.replace("__TITLE__", title).replace("__DATA__", payload) @@ -96,8 +96,8 @@ def render_page(trials: list[Trial], *, out_dir: Path) -> str: #stage { display: grid; grid-template-columns: repeat(auto-fit, minmax(340px, 1fr)); gap: 16px; padding: 16px; } .trial { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: 12px 14px; display: flex; flex-direction: column; gap: 10px; } .trial h2 { margin: 0; font-size: 15px; display: flex; gap: 10px; align-items: baseline; } - .trial h2 .slot { text-transform: uppercase; letter-spacing: .04em; font-size: 12px; padding: 2px 8px; border-radius: 999px; color: #fff; } - .slot.jev { background: var(--jev); } .slot.llm { background: var(--llm); } .slot.other { background: var(--muted); } + .trial h2 .model { text-transform: uppercase; letter-spacing: .04em; font-size: 12px; padding: 2px 8px; border-radius: 999px; color: #fff; } + .model.jev { background: var(--jev); } .model.llm { background: var(--llm); } .model.other { background: var(--muted); } .facts { color: var(--muted); font-size: 12px; display: flex; gap: 12px; flex-wrap: wrap; } .board { min-height: 180px; display: flex; align-items: center; justify-content: center; } .board img { max-width: 100%; max-height: 420px; border: 1px solid var(--line); border-radius: 6px; } @@ -155,7 +155,7 @@ def render_page(trials: list[Trial], *, out_dir: Path) -> str: const stage = document.getElementById("stage"); const slider = document.getElementById("slider"); const clock = document.getElementById("clock"); - document.getElementById("title").textContent = data.eval + " · seed " + data.trials[0].seed + " · " + data.trials.map(t => t.slot).join(" vs "); + document.getElementById("title").textContent = data.eval + " · seed " + data.trials[0].seed + " · " + data.trials.map(t => t.model).join(" vs "); slider.max = maxT; const esc = s => String(s == null ? "" : s).replace(/[&<>"]/g, c => ({"&":"&","<":"<",">":">",'"':"""}[c])); @@ -240,11 +240,11 @@ def render_page(trials: list[Trial], *, out_dir: Path) -> str: const k = step[i], view = trial.views[Math.min(k, trial.views.length - 1)] || {state: {}, candidates: {}}; const decision = trial.decisions[k]; const draw = trial.front === "browser" ? drawBrowser : (drawers[data.eval] || drawGeneric); - const slotClass = trial.slot === "jev" || trial.slot === "llm" ? trial.slot : "other"; - const badge = trial.slot === "llm" ? "LLM" : "System 1 · " + trial.slot; + const modelClass = trial.model === "jev" || trial.model === "llm" ? trial.model : "other"; + const badge = trial.model === "llm" ? "LLM" : "System 1 · " + trial.model; const action = decision ? 'step ' + (k + 1) + ' of ' + trial.steps + ': ' + esc(decision.key) + '' + esc(decision.ms) + ' ms' + (decision.confidence ? ", confidence " + decision.confidence.toFixed(2) : "") + '' : 'final · score ' + esc(trial.score) + ''; - return '

' + esc(badge) + ' score ' + esc(trial.score) + '

' + + return '

' + esc(badge) + ' score ' + esc(trial.score) + '

' + '
' + trial.elapsed_s + ' s' + trial.steps + ' steps' + trial.decisions.length + ' decisions' + money(trial.cost_usd) + '
' + '
' + draw(view, trial, k) + '
' + action + bars(decision, view) + '
t = ' + secs(trial.times[k]) + ' of ' + secs(trial.times[trial.times.length - 1]) + '
'; diff --git a/evals/replay/trial.py b/evals/replay/trial.py index 1fc6884..a7ed5b6 100644 --- a/evals/replay/trial.py +++ b/evals/replay/trial.py @@ -8,14 +8,14 @@ from pathlib import Path from typing import Any -from evals.table import slot_label, window_s +from evals.table import model_label, window_s @dataclass(frozen=True) class Trial: path: Path eval_name: str - slot: str + model: str seed: int score: float elapsed_s: float @@ -50,7 +50,7 @@ def read_trial(path: Path) -> Trial: return Trial( path=path, eval_name=str(result.get("source") or path.parent.parent.name), - slot=slot_label(result), + model=model_label(result), seed=int(seed_text) if seed_text.isdigit() else 0, score=float(((result.get("verifier_result") or {}).get("rewards") or {}).get("reward") or 0.0), elapsed_s=float(metadata.get("elapsed_s") or window_s(result)), @@ -66,8 +66,8 @@ def read_trial(path: Path) -> Trial: def read_browser_run(path: Path) -> Trial: - """A browser run's logs folder as a trial: ``answer.json`` (the run's result with its agent and slot), - ``decision_ticks.json`` for a decision-model slot or ``chat_calls.json`` for the chat model, and the frames + """A browser run's logs folder as a trial: ``answer.json`` (the run's result with its agent and model), + ``decision_ticks.json`` for a decision model or ``chat_calls.json`` for the chat model, and the frames ``evals.replay.cast`` saved under ``frames/``. A decision-model run's steps sit on the policy's own clock (each tick's ``elapsed_ms``); a chat-model run's @@ -75,9 +75,10 @@ def read_browser_run(path: Path) -> Trial: second of the frames' clock, the MCP session's. """ answer = json.loads((path / "answer.json").read_text(encoding="utf-8")) - slot, elapsed_ms = str(answer["slot"]), int(answer["elapsed_ms"]) + model = str(answer.get("model") or answer["slot"]) # runs written by 0.1.0 name the model under "slot" + elapsed_ms = int(answer["elapsed_ms"]) extra: dict[str, Any] = {"front": "browser"} - if slot == "llm": + if model == "llm": calls = json.loads((path / "chat_calls.json").read_text(encoding="utf-8")) acted = [call for call in calls if call["tool_calls"]] decisions = [_chat_decision(step, call) for step, call in enumerate(acted, start=1)] @@ -85,7 +86,7 @@ def read_browser_run(path: Path) -> Trial: else: record = json.loads((path / "decision_ticks.json").read_text(encoding="utf-8")) ticks = list(record["ticks"]) - decisions = [_tick_decision(tick, slot) for tick in ticks] + decisions = [_tick_decision(tick, model) for tick in ticks] candidates = [dict(tick.get("candidates") or {}) for tick in ticks] extra["times"] = [0] + [int(tick["elapsed_ms"]) for tick in ticks[:-1]] + [elapsed_ms] if ticks else [0] extra["history"] = list((record.get("report") or {}).get("history") or []) @@ -102,7 +103,7 @@ def read_browser_run(path: Path) -> Trial: return Trial( path=path, eval_name=str(answer["agent"]), - slot=slot, + model=model, seed=0, score=score, elapsed_s=round(elapsed_ms / 1000, 1), @@ -117,7 +118,7 @@ def read_browser_run(path: Path) -> Trial: ) -def _tick_decision(tick: dict[str, Any], slot: str) -> dict[str, Any]: +def _tick_decision(tick: dict[str, Any], model: str) -> dict[str, Any]: target = str(tick.get("target") or "") key = f"{tick['operation']} · {target}" if target else str(tick["operation"]) return { @@ -126,7 +127,7 @@ def _tick_decision(tick: dict[str, Any], slot: str) -> dict[str, Any]: "ms": int(tick["decision_ms"]), "confidence": float(tick["confidence"]), "probabilities": dict(tick.get("probabilities") or {}), - "source": slot, + "source": model, } diff --git a/evals/table.py b/evals/table.py index 50748ef..31d9926 100644 --- a/evals/table.py +++ b/evals/table.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""One table over every job folder: per eval and slot, score, seconds, steps, decisions, chat calls and dollars. +"""One table over every job folder: per eval and model, score, seconds, steps, decisions, chat calls and dollars. ``python -m evals.table evals/results`` reads the job folders that ``write_job`` produces, one ``result.json`` per trial. Rows are keyed by the results root's child folder; a trial with ``exception_info`` is left out and counted in the errors column. @@ -18,13 +18,13 @@ from s1a.jobs import BOOTSTRAP_RESAMPLES, bootstrap_interval -COLUMNS = ("eval", "slot", "N", "errors", "score", "s / episode", "steps", "decisions", "chat calls", "$ / episode") +COLUMNS = ("eval", "model", "N", "errors", "score", "s / episode", "steps", "decisions", "chat calls", "$ / episode") @dataclass(frozen=True) class Trial: eval_name: str - slot: str + model: str errored: bool # result.json holds exception_info: no score, counted in the errors column only score: float elapsed_s: float @@ -43,8 +43,8 @@ def window_s(result: dict[str, Any]) -> float: return (datetime.fromisoformat(finished) - datetime.fromisoformat(started)).total_seconds() -def slot_label(result: dict[str, Any]) -> str: - """The slot a trial's ``result.json`` names: ``jev``, ``llm``, ``random`` or a rule name, after any ``/`` prefix.""" +def model_label(result: dict[str, Any]) -> str: + """The model a trial's ``result.json`` names: ``jev``, ``llm``, ``random`` or a rule name, after any ``/`` prefix.""" name = str((result.get("agent_info") or {}).get("name") or "") return name.partition("/")[2] or name @@ -60,7 +60,7 @@ def read_trial(trial_dir: Path, *, eval_name: str) -> Trial | None: steps = metadata.get("steps") return Trial( eval_name=eval_name, - slot=slot_label(result), + model=model_label(result), errored=result.get("exception_info") is not None, score=float(((result.get("verifier_result") or {}).get("rewards") or {}).get("reward") or 0.0), elapsed_s=float(metadata.get("elapsed_s") or window_s(result)), @@ -82,13 +82,13 @@ def read_results(root: Path) -> list[Trial]: def rows(trials: list[Trial]) -> list[dict[str, Any]]: - """One row per (eval, slot): N and errors, then the mean score with its 95 % bootstrap interval, medians and - means per played episode; a slot whose every trial errored has no score.""" + """One row per (eval, model): N and errors, then the mean score with its 95 % bootstrap interval, medians and + means per played episode; a model whose every trial errored has no score.""" groups: dict[tuple[str, str], list[Trial]] = {} for trial in trials: - groups.setdefault((trial.eval_name, trial.slot), []).append(trial) + groups.setdefault((trial.eval_name, trial.model), []).append(trial) table = [] - for (eval_name, slot), all_members in sorted(groups.items()): + for (eval_name, model), all_members in sorted(groups.items()): members = [member for member in all_members if not member.errored] scores = [member.score for member in members] costs = [member.cost_usd for member in members] @@ -96,7 +96,7 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: table.append( { "eval": eval_name, - "slot": slot, + "model": model, "N": len(members), "errors": len(all_members) - len(members), "mean_score": round(statistics.mean(scores), 3) if scores else None, @@ -125,7 +125,7 @@ def markdown(table: list[dict[str, Any]]) -> str: for key in ("median_s", "mean_steps", "mean_decisions", "mean_chat_calls") ] lines.append( - f"| {row['eval']} | {row['slot']} | {row['N']} | {row['errors']} | {score} | " + f"| {row['eval']} | {row['model']} | {row['N']} | {row['errors']} | {score} | " f"{cells[0]} | {cells[1]} | {cells[2]} | {cells[3]} | {cost} |" ) return "\n".join(lines) diff --git a/evals/ticket_router/RESULTS.md b/evals/ticket_router/RESULTS.md index b1d8659..cc82ace 100644 --- a/evals/ticket_router/RESULTS.md +++ b/evals/ticket_router/RESULTS.md @@ -44,15 +44,15 @@ From the repository checkout after installing the core and dev dependencies: ```sh uv run --no-sync pytest tests/test_agents_ticket_router.py -q uv run --no-sync s1a probe evals/ticket_router/probe.jsonl -uv run --no-sync s1a run ticket_router --slot random --rethink off --episodes 3 --seed 0 -uv run --no-sync s1a run ticket_router --slot rule --rethink off --episodes 3 --seed 0 +uv run --no-sync s1a run ticket_router --model random --rethink off --episodes 3 --seed 0 +uv run --no-sync s1a run ticket_router --model rule --rethink off --episodes 3 --seed 0 ``` Once the English probe passes (at least 10/12), run matched small model comparisons: ```sh -uv run --no-sync s1a run ticket_router --slot jev --rethink off --episodes 3 --seed 0 --log -uv run --no-sync s1a run ticket_router --slot llm --rethink off --episodes 3 --seed 0 +uv run --no-sync s1a run ticket_router --model jev --rethink off --episodes 3 --seed 0 --log +uv run --no-sync s1a run ticket_router --model llm --rethink off --episodes 3 --seed 0 ``` Each episode is a batch, not one ticket. The default batch contains thirty tickets. The model commands require diff --git a/pyproject.toml b/pyproject.toml index 5035701..682ddda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,8 +40,8 @@ alfworld-visual = [ "torchvision>=0.15", ] report = ["pillow>=10", "playwright>=1.45"] # evals.replay: pages, GIFs; never imported by a runner -laya = ["laya>=0.3.4"] # the in-process decision model behind --slot laya; pulls torch and transformers -cua = [ # Cua-S1 Nano behind --slot cua; pinned to the Cua PR that ships the checkpoints (trycua/cua#4023), pulls torch +laya = ["laya>=0.3.4"] # the in-process decision model behind --model laya; pulls torch and transformers +cua = [ # Cua-S1 Nano behind --model cua; pinned to the Cua PR that ships the checkpoints (trycua/cua#4023), pulls torch "cua-s1 @ git+https://github.com/trycua/cua.git@aea61b6eb97e2d8c0f6f71eb804e5769fe910af4#subdirectory=libs/cua-s1/python", "huggingface-hub>=0.24", ] diff --git a/s1a/agents/_templates/browser_agent.py b/s1a/agents/_templates/browser_agent.py index 1fcbd08..c766e17 100644 --- a/s1a/agents/_templates/browser_agent.py +++ b/s1a/agents/_templates/browser_agent.py @@ -2,7 +2,7 @@ """The template for a browser-front agent: copy this file to ``s1a/agents/.py``. Change the name, the description and the goal; keep the shipped operation rules until a site family shows that it -needs its own, then write that string here. ``s1a run --slot jev --goal "..."`` runs one task; the +needs its own, then write that string here. ``s1a run --model jev --goal "..."`` runs one task; the policy switches (``--batch``, ``--prefetch``, ``--goal-values``) are run-time flags, not spec fields. """ diff --git a/s1a/agents/_templates/tool_agent.py b/s1a/agents/_templates/tool_agent.py index ae96adf..e55e0ec 100644 --- a/s1a/agents/_templates/tool_agent.py +++ b/s1a/agents/_templates/tool_agent.py @@ -2,8 +2,8 @@ """Nim, the template for a tool-front agent: copy this file to ``s1a/agents/.py`` and replace each part. Two players alternately take 1, 2 or 3 stones from one pile; whoever takes the last stone wins. The agent moves first -and a fixed opponent answers with the winning reply whenever one exists. ``s1a run --slot random ---rethink off --episodes 3`` plays it without any key; ``--slot rule`` plays the winning strategy. +and a fixed opponent answers with the winning reply whenever one exists. ``s1a run --model random +--rethink off --episodes 3`` plays it without any key; ``--model rule`` plays the winning strategy. """ from __future__ import annotations diff --git a/s1a/agents/alfworld.py b/s1a/agents/alfworld.py index 482f92a..0a6194d 100644 --- a/s1a/agents/alfworld.py +++ b/s1a/agents/alfworld.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""ALFWorld text games (TextWorld): ``s1a run alfworld --slot jev --rethink on --episodes 10``. +"""ALFWorld text games (TextWorld): ``s1a run alfworld --model jev --rethink on --episodes 10``. Needs the ``alfworld`` and ``textworld`` packages and ``ALFWORLD_DATA`` (Python 3.11; see the README). """ @@ -198,7 +198,7 @@ def make_series(flags: argparse.Namespace) -> Series: f"--episodes {flags.episodes} but only {len(indices)} games from --offset {flags.offset} with " f"--stride {flags.stride} ({len(files)} solvable games)" ) - env = AlfworldEnv([files[index] for index in indices], flags.max_steps, every_command=flags.slot == "rule") + env = AlfworldEnv([files[index] for index in indices], flags.max_steps, every_command=flags.model == "rule") def annotate(_env: Env, episode: Episode) -> None: episode.extra["game"] = env.game_name diff --git a/s1a/agents/allrecipes.py b/s1a/agents/allrecipes.py index 2500add..f00d60c 100644 --- a/s1a/agents/allrecipes.py +++ b/s1a/agents/allrecipes.py @@ -1,7 +1,7 @@ # coding: utf-8 -"""Allrecipes, WebVoyager task Allrecipes--0: ``s1a run allrecipes --slot jev --batch on --headed``. +"""Allrecipes, WebVoyager task Allrecipes--0: ``s1a run allrecipes --model jev --batch on --headed``. -The browser-use demo against the chat model in the same slot (``--slot llm``). ``TASK`` is task ``Allrecipes--0`` of +The browser-use demo against the chat model in the same model_name (``--model llm``). ``TASK`` is task ``Allrecipes--0`` of the WebVoyager task set (MinorJerry/WebVoyager, He et al. 2024, Apache License 2.0; see NOTICE), and the goal wraps it in the WebVoyager runner's instruction shape. The run is headed because Allrecipes answers a headless Chromium with a bot wall. diff --git a/s1a/agents/blackjack.py b/s1a/agents/blackjack.py index ec35e8f..a5943e4 100644 --- a/s1a/agents/blackjack.py +++ b/s1a/agents/blackjack.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""Blackjack (RLCard): ``s1a run blackjack --slot jev --rethink off --episodes 100``.""" +"""Blackjack (RLCard): ``s1a run blackjack --model jev --rethink off --episodes 100``.""" from __future__ import annotations diff --git a/s1a/agents/desktop.py b/s1a/agents/desktop.py index 15996bb..1727a03 100644 --- a/s1a/agents/desktop.py +++ b/s1a/agents/desktop.py @@ -4,13 +4,13 @@ The Calculator example:: s1a run desktop --app Calculator --goal "compute 12 times 7" --expect 84 --execute \\ - --plan "1,2,Multiply|×,7,Equals|=" --clear "All Clear" --slot jev --rethink off --episodes 1 + --plan "1,2,Multiply|×,7,Equals|=" --clear "All Clear" --model jev --rethink off --episodes 1 On Windows use ``--app "Windows Calculator" --expect "Display is 84"`` and match the UIA button labels with ``--plan "One,Two,Multiply by,Seven,Equals" --clear Clear``. Result text is matched exactly, in the app's language. Without ``--execute`` the run is a dry run: one decision, recorded as ``planned``, nothing clicked. ``--plan`` is the -rule baseline (``--slot rule``): button labels in order, ``|`` between variants of one label. Needs ``cua-driver`` on +rule baseline (``--model rule``): button labels in order, ``|`` between variants of one label. Needs ``cua-driver`` on PATH; macOS also needs Accessibility and Screen Recording granted. """ @@ -98,9 +98,7 @@ def make_series(flags: argparse.Namespace) -> Series: def flags(parser: argparse.ArgumentParser) -> None: parser.add_argument("--app", required=True, help="app name (Windows Calculator / Calculator), or a Windows AUMID") - parser.add_argument( - "--goal", required=True, help="what to do in the window, read by the model in the slot on every turn" - ) + parser.add_argument("--goal", required=True, help="what to do in the window, read by the model on every turn") parser.add_argument("--expect", required=True, help="the text a display or label shows when the goal is met") parser.add_argument("--execute", action="store_true", help="click for real; without it one decision is planned") parser.add_argument("--plan", default="", help="the rule baseline: button labels in order, | between variants") diff --git a/s1a/agents/flights.py b/s1a/agents/flights.py index 946e167..1c7fc38 100644 --- a/s1a/agents/flights.py +++ b/s1a/agents/flights.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""Google Flights, one-way Zurich to London: ``s1a run flights --slot jev --batch on --profile-out run.json``. +"""Google Flights, one-way Zurich to London: ``s1a run flights --model jev --batch on --profile-out run.json``. The timing demo against browser-use/jev-ultrafast; the clock runs from the first decision to the final DONE. ``--batch on`` is the measured best arm: 10.4 s, 3 of 3 verified (docs/benchmarks.md). diff --git a/s1a/agents/game2048.py b/s1a/agents/game2048.py index 69e98e2..4cdda60 100644 --- a/s1a/agents/game2048.py +++ b/s1a/agents/game2048.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""2048 (the original MIT game, self-hosted): ``s1a run game2048 --slot jev --rethink on --episodes 10``.""" +"""2048 (the original MIT game, self-hosted): ``s1a run game2048 --model jev --rethink on --episodes 10``.""" from __future__ import annotations @@ -43,7 +43,7 @@ } """ # The game draws every new tile from Math.random; a seeded mulberry32 in its place makes an episode's tile -# sequence a function of the seed and the moves, so every slot plays the same boards. +# sequence a function of the seed and the moves, so every model plays the same boards. SEED_RANDOM = """ (seed) => { let s = seed >>> 0; diff --git a/s1a/agents/millionaire.py b/s1a/agents/millionaire.py index 1eb49a5..a456235 100644 --- a/s1a/agents/millionaire.py +++ b/s1a/agents/millionaire.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""Who Wants to Be a Millionaire (self-hosted page, Open Trivia DB): ``s1a run millionaire --slot jev --rethink off --episodes 5``.""" +"""Who Wants to Be a Millionaire (self-hosted page, Open Trivia DB): ``s1a run millionaire --model jev --rethink off --episodes 5``.""" from __future__ import annotations diff --git a/s1a/agents/ticket_router.py b/s1a/agents/ticket_router.py index 61769c1..d7f4f57 100644 --- a/s1a/agents/ticket_router.py +++ b/s1a/agents/ticket_router.py @@ -104,7 +104,7 @@ def report(self) -> dict[str, Any]: support = Counter(row["label"] for row in self._batch) hits = Counter(row["expected"] for row in self._results if row["correct"]) total = len(self._batch) - # Fingerprint includes state and labels, permitting checks that all slots saw identical data. + # Fingerprint includes state and labels, permitting checks that all models saw identical data. fingerprint = hashlib.sha256( json.dumps(self._batch, ensure_ascii=False, sort_keys=True).encode("utf-8") ).hexdigest() diff --git a/s1a/browser/browse.py b/s1a/browser/browse.py index a57d1ff..b9be304 100644 --- a/s1a/browser/browse.py +++ b/s1a/browser/browse.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""The browser front: one task through openJiuwen's browser subagent with a decision model, or the chat model, in its slot.""" +"""The browser front: one task through openJiuwen's browser subagent with a decision model, or the chat model, in its model slot.""" from __future__ import annotations @@ -34,7 +34,7 @@ from s1a.spec import BrowserAgentSpec, positive_float, positive_int Answer = dict[str, Any] -BROWSER_SLOTS = ( +BROWSER_MODEL_NAMES = ( "jev", "laya", "cua", @@ -73,9 +73,9 @@ def finish_llm(answer: Answer) -> Answer: return answer -def finish_decision_model(answer: Answer, *, slot: str) -> Answer: +def finish_decision_model(answer: Answer, *, model_name: str) -> Answer: """The policy's answer from the page it reached (on DONE, or on BLOCKED after progress) is the result; none fails. - ``slot`` names the decision model in the error.""" + ``model_name`` names the decision model in the error.""" summary = terminal_summary(answer["final"]) if summary is None: return answer @@ -87,9 +87,9 @@ def finish_decision_model(answer: Answer, *, slot: str) -> Answer: if answer["ok"]: answer["error"] = None # run_task flagged the harness's own verdict; the policy's answer is the one that counts elif status == "DONE": - answer["error"] = f"{slot} DONE without an answer" + answer["error"] = f"{model_name} DONE without an answer" else: - answer["error"] = f"{slot} {status}: {summary.get('reason') or 'no reason given'}" + answer["error"] = f"{model_name} {status}: {summary.get('reason') or 'no reason given'}" return answer @@ -142,7 +142,7 @@ async def browse( spec: BrowserAgentSpec, policy: BrowserPolicy, *, - slot: str, + model_name: str, goal: str, timeout_s: float, max_steps: int, @@ -154,19 +154,19 @@ async def browse( """One task with a decision model (``jev``, ``laya``, ``cua``) or the chat model (``llm``) deciding every browser step. Needs a started Runner. - A decision-model slot writes ``decision_ticks.json`` under ``logs_dir`` and returns the ticks and the policy's report with - the answer; the llm slot writes ``chat_calls.json``. ``decision_model`` is required by the decision-model slots and unused - by the llm slot. + A decision model writes ``decision_ticks.json`` under ``logs_dir`` and returns the ticks and the policy's report with + the answer; ``llm`` writes ``chat_calls.json``. ``decision_model`` is required by every other name and unused + by ``llm``. """ logs_dir.mkdir(parents=True, exist_ok=True) calls: list[dict[str, Any]] = [] counted = CountingModel(chat, calls) workspace = str(logs_dir / "workspace") # the harness scaffolds SOUL.md, memory/ and friends here, not in the cwd instance = BrowserInstanceConfig(launch_args=browser_launch_args(headless)) - match slot: + match model_name: case "jev" | "laya" | "cua": if decision_model is None: - raise RuntimeError(f"the {slot} slot needs a decision model") + raise RuntimeError(f"--model {model_name} needs a decision model") slot_model = BrowserDecisionModel(spec, policy, counted, decision_model=decision_model, value_model=None) agent = create_browser_agent( slot_model, @@ -195,7 +195,7 @@ async def browse( ), encoding="utf-8", ) - return finish_decision_model(answer, slot=decision_model.name) + return finish_decision_model(answer, model_name=decision_model.name) case "llm": # create_browser_agent swaps a plain Model for a fresh copy at its own temperature; the marker keeps the counter in setattr(counted, _BROWSER_MODEL_TEMPERATURE_MARKER, DEFAULT_BROWSER_AGENT_TEMPERATURE) @@ -214,17 +214,17 @@ async def browse( (logs_dir / "chat_calls.json").write_text(json.dumps(calls, ensure_ascii=False, indent=1), encoding="utf-8") return answer case _: - raise ValueError(f"unknown slot {slot!r}; one of {BROWSER_SLOTS}") + raise ValueError(f"unknown model {model_name!r}; one of {BROWSER_MODEL_NAMES}") def parser(spec: BrowserAgentSpec) -> argparse.ArgumentParser: """The shared browser flags: the spec's budget and goal as defaults, the policy switches off except prefetch.""" build = argparse.ArgumentParser(prog=f"s1a run {spec.name}", description=spec.description) build.add_argument( - "--slot", - choices=BROWSER_SLOTS, + "--model", + choices=BROWSER_MODEL_NAMES, required=True, - help="who decides each browser step: jev (over HTTP), laya or cua (in process), or llm (the chat model)", + help="who decides each browser step: jev (over HTTP), laya or cua (in process), or llm (the chat model in MODEL_NAME)", ) build.add_argument( "--goal", default=spec.goal, required=spec.goal is None, help="the task; the spec's goal when it has one" @@ -243,19 +243,19 @@ def parser(spec: BrowserAgentSpec) -> argparse.ArgumentParser: "--batch", choices=("on", "off"), default="off", - help="decision-model slots: one browser_run_code_unsafe call per step (on, needs unsafe_dev) or the browser_* tools (off)", + help="decision models only: one browser_run_code_unsafe call per step (on, needs unsafe_dev) or the browser_* tools (off)", ) build.add_argument( "--prefetch", choices=("on", "off"), default="on", - help="decision-model slots: generate a typed value for every editable field as soon as a probe shows it", + help="decision models only: generate a typed value for every editable field as soon as a probe shows it", ) build.add_argument( "--goal-values", choices=("on", "off"), default="off", - help="decision-model slots: offer values extracted from the goal as a choice head", + help="decision models only: offer values extracted from the goal as a choice head", ) build.add_argument( "--logs-dir", @@ -267,7 +267,7 @@ def parser(spec: BrowserAgentSpec) -> argparse.ArgumentParser: "--profile-out", type=Path, default=None, - help="decision-model slots: attach the profiler and write its JSON here", + help="decision models only: attach the profiler and write its JSON here", ) return build @@ -282,16 +282,16 @@ def policy_from_args(args: argparse.Namespace) -> BrowserPolicy: async def play(spec: BrowserAgentSpec, args: argparse.Namespace) -> Answer: """One task inside an already started Runner: the answer without the ticks, which ``decision_ticks.json`` holds.""" - if args.profile_out is not None and args.slot == "llm": - raise RuntimeError("--profile-out needs a decision-model slot: the profiler times the slot model's turns") + if args.profile_out is not None and args.model == "llm": + raise RuntimeError("--profile-out needs a decision model: the profiler times the slot model's turns") chat = chat_model_from_env() - decision_model = None if args.slot == "llm" else build_model(args.slot) + decision_model = None if args.model == "llm" else build_model(args.model) profiler = BrowserProfiler().attach() if args.profile_out is not None else None try: answer = await browse( spec, policy_from_args(args), - slot=args.slot, + model_name=args.model, goal=args.goal, timeout_s=args.timeout, max_steps=args.max_steps, @@ -308,8 +308,8 @@ async def play(spec: BrowserAgentSpec, args: argparse.Namespace) -> Answer: result = {key: value for key, value in answer.items() if key != "ticks"} ( args.logs_dir / "answer.json" - ).write_text( # the printed result, with the agent and the slot, next to the run's records - json.dumps({**result, "agent": spec.name, "slot": args.slot}, ensure_ascii=False, indent=1), encoding="utf-8" + ).write_text( # the printed result, with the agent and the model, next to the run's records + json.dumps({**result, "agent": spec.name, "model": args.model}, ensure_ascii=False, indent=1), encoding="utf-8" ) if profiler is not None: final_url = str((answer.get("terminal") or {}).get("url") or "") @@ -321,7 +321,7 @@ async def play(spec: BrowserAgentSpec, args: argparse.Namespace) -> Answer: final_url=final_url, verified_prefix=site, ) - profile["slot"], profile["usage"] = args.slot, answer["usage"] + profile["model"], profile["usage"] = args.model, answer["usage"] print(render(profile), file=sys.stderr) args.profile_out.write_text(json.dumps(profile, ensure_ascii=False, indent=1), encoding="utf-8") return result diff --git a/s1a/browser/decision_model.py b/s1a/browser/decision_model.py index 0ba0b7a..001478b 100644 --- a/s1a/browser/decision_model.py +++ b/s1a/browser/decision_model.py @@ -9,7 +9,7 @@ the bound ``BrowserAgentRuntime``, asks the model (every head of the action space in one ``decide_many``, re-asked once on an unusable answer), and returns exactly one ``browser_*`` tool call. Every other call (summaries, value generation) goes to the wrapped chat model. Jev over HTTP and Laya in process fill the -slot alike. +model_name alike. """ from __future__ import annotations @@ -138,7 +138,7 @@ def __init__( sends each action and the next probe as one ``browser_run_code_unsafe`` call when that tool is offered, skipping the runtime's target validation and one transport round trip per step. ``value_model`` answers the typed-value calls when given; the fallback chat model otherwise. - ``decision_model`` decides every browser step; its ``name`` (``jev``, ``laya``) is the slot.""" + ``decision_model`` decides every browser step; its ``name`` (``jev``, ``laya``) is the ``--model`` value.""" super().__init__(fallback.model_client_config, fallback.model_config) self._spec = spec self._fallback = fallback diff --git a/s1a/cli.py b/s1a/cli.py index cac642a..d049de0 100644 --- a/s1a/cli.py +++ b/s1a/cli.py @@ -19,7 +19,7 @@ from s1a.run import started_runner from s1a.spec import Json -DECIDE_SLOTS = ( +DECIDE_MODEL_NAMES = ( "jev", "laya", "cua", @@ -45,7 +45,7 @@ def parser() -> argparse.ArgumentParser: ) decide.add_argument("--rules", required=True, help="the facts the model applies when it picks") decide.add_argument( - "--slot", choices=DECIDE_SLOTS, default="jev", help="who answers: jev, or laya and cua in process" + "--model", choices=DECIDE_MODEL_NAMES, default="jev", help="who answers: jev, or laya and cua in process" ) fit = commands.add_parser("probe", help="the fit probe: hand-written choice cases from a JSONL file") fit.add_argument( @@ -53,7 +53,9 @@ def parser() -> argparse.ArgumentParser: type=Path, help="JSONL, one case per line: state (object), options (key to text), rules, accept (list of right keys), note", ) - fit.add_argument("--slot", choices=DECIDE_SLOTS, default="jev", help="who answers: jev, or laya and cua in process") + fit.add_argument( + "--model", choices=DECIDE_MODEL_NAMES, default="jev", help="who answers: jev, or laya and cua in process" + ) return build @@ -87,9 +89,9 @@ async def run_agent(name: str, flags: list[str]) -> Json: async def decide(args: argparse.Namespace) -> dict[str, Any]: - """One question through the slot's model; prints ``{"choice", "probabilities", "confidence", "ms"}``.""" + """One question through the model ``--model`` names; prints ``{"choice", "probabilities", "confidence", "ms"}``.""" state, options = parse_state(args.state), parse_options(args.option) # bad input is reported before any key check - decision_model = build_model(args.slot) + decision_model = build_model(args.model) try: answer = await probe.pick(decision_model, state=state, options=options, rules=args.rules) finally: @@ -98,9 +100,9 @@ async def decide(args: argparse.Namespace) -> dict[str, Any]: return answer -async def run_probe(cases: Path, slot: str) -> dict[str, Any]: +async def run_probe(cases: Path, model_name: str) -> dict[str, Any]: read = probe.read_cases(cases) - decision_model = build_model(slot) + decision_model = build_model(model_name) try: summary = await probe.run(read, decision_model) finally: @@ -125,7 +127,7 @@ def main(argv: list[str]) -> int: case "decide": asyncio.run(decide(args)) case "probe": - return 0 if asyncio.run(run_probe(args.cases, args.slot))["verdict"] == "fits" else 1 + return 0 if asyncio.run(run_probe(args.cases, args.model))["verdict"] == "fits" else 1 except (agents.UnknownAgent, ValueError) as exc: # JSONDecodeError is a ValueError print(exc, file=sys.stderr) return 2 diff --git a/s1a/config.py b/s1a/config.py index d9602f6..c416450 100644 --- a/s1a/config.py +++ b/s1a/config.py @@ -70,7 +70,7 @@ def chat_model_from_env() -> Model: def optional_chat_model() -> Model | None: - """The chat model when the environment names one; None otherwise (the slots that need it raise).""" + """The chat model when the environment names one; None otherwise (the models that need it raise).""" try: return chat_model_from_env() except RuntimeError: diff --git a/s1a/decision_models/__init__.py b/s1a/decision_models/__init__.py index 51c981c..bf68c9f 100644 --- a/s1a/decision_models/__init__.py +++ b/s1a/decision_models/__init__.py @@ -10,7 +10,7 @@ from s1a.decision_models.base import DecisionModel, JevTransport from s1a.decision_models.baselines import RandomModel, Rule, RuleModel from s1a.decision_models.cua import CuaS1Model -from s1a.decision_models.factory import DECISION_MODEL_SLOTS, build_model +from s1a.decision_models.factory import DECISION_MODEL_NAMES, build_model from s1a.decision_models.fakes import ScriptedModel, ScriptedTransport from s1a.decision_models.jev import JevModel, jev_question from s1a.decision_models.laya import LayaModel @@ -31,7 +31,7 @@ from s1a.decision_models.validation import choice_faults, validate_answers, validate_choice, validate_noul __all__ = [ - "DECISION_MODEL_SLOTS", + "DECISION_MODEL_NAMES", "Answer", "DecisionModel", "JevTransport", diff --git a/s1a/decision_models/base.py b/s1a/decision_models/base.py index 6534139..b120ae0 100644 --- a/s1a/decision_models/base.py +++ b/s1a/decision_models/base.py @@ -40,7 +40,7 @@ async def close(self) -> None: ... class DecisionModel(ABC): """A model that reads an observation and a discrete action space and returns a distribution over it.""" - name: str = "decision_model" # the slot name; lands in every tick's ``source`` + name: str = "decision_model" # the ``--model`` value; lands in every tick's ``source`` supports_images: bool = False question_types: frozenset[str] = frozenset({"choice", "noul"}) deterministic: bool = False # the same request always gets the same answer, so a re-ask is a wasted call diff --git a/s1a/decision_models/cua.py b/s1a/decision_models/cua.py index b2fddf9..cc7d2e7 100644 --- a/s1a/decision_models/cua.py +++ b/s1a/decision_models/cua.py @@ -143,7 +143,7 @@ def from_env(cls) -> "CuaS1Model": from cua_s1.nano import load_nano_checkpoint except ImportError as exc: raise build_error( - StatusCode.MODEL_SERVICE_CONFIG_ERROR, error_msg="the cua slot needs the cua extra: uv sync --extra cua" + StatusCode.MODEL_SERVICE_CONFIG_ERROR, error_msg="--model cua needs the cua extra: uv sync --extra cua" ) from exc checkpoint = os.getenv("CUA_S1_CHECKPOINT") or CUA_DEFAULT_CHECKPOINT subfolder = os.getenv("CUA_S1_SUBFOLDER") or CUA_DEFAULT_SUBFOLDER diff --git a/s1a/decision_models/factory.py b/s1a/decision_models/factory.py index fb8f5df..d81ae45 100644 --- a/s1a/decision_models/factory.py +++ b/s1a/decision_models/factory.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""a decision model by slot name, from the environment.""" +"""A decision model by name, from the environment.""" from __future__ import annotations @@ -9,18 +9,18 @@ from s1a.decision_models.jev import JevModel from s1a.decision_models.laya import LayaModel -DECISION_MODEL_SLOTS = ( +DECISION_MODEL_NAMES = ( "jev", "laya", "cua", "random", "rule", -) # the slots a decision model fills; ``llm`` is not a decision model +) # the names that build a decision model; ``llm`` is not one -def build_model(slot: str, *, seed: int = 0, rule: tuple[str, Rule] | None = None) -> DecisionModel: +def build_model(model_name: str, *, seed: int = 0, rule: tuple[str, Rule] | None = None) -> DecisionModel: """``jev``, ``laya`` and ``cua`` from the environment, ``random`` from the seed, ``rule`` from the agent's baseline.""" - match slot: + match model_name: case "jev": return JevModel.from_env() case "laya": @@ -34,4 +34,4 @@ def build_model(slot: str, *, seed: int = 0, rule: tuple[str, Rule] | None = Non raise RuntimeError("this agent has no rule baseline") return RuleModel(*rule) case _: - raise ValueError(f"unknown decision-model slot {slot!r}; one of {DECISION_MODEL_SLOTS}") + raise ValueError(f"unknown decision model {model_name!r}; one of {DECISION_MODEL_NAMES}") diff --git a/s1a/decision_models/laya.py b/s1a/decision_models/laya.py index c2ff231..1fb786d 100644 --- a/s1a/decision_models/laya.py +++ b/s1a/decision_models/laya.py @@ -94,7 +94,7 @@ def from_env(cls) -> "LayaModel": except ImportError as exc: raise build_error( StatusCode.MODEL_SERVICE_CONFIG_ERROR, - error_msg="the laya slot needs the laya extra: uv sync --extra laya", + error_msg="--model laya needs the laya extra: uv sync --extra laya", ) from exc model = os.getenv("LAYA_MODEL") or LAYA_DEFAULT_MODEL subfolder = os.getenv("LAYA_SUBFOLDER") or None @@ -108,7 +108,7 @@ def from_env(cls) -> "LayaModel": StatusCode.MODEL_SERVICE_CONFIG_ERROR, error_msg=( f"laya {version} loaded an agent without a callable system_one(state, questions); " - "the s1a laya slot needs that method" + "the s1a laya model needs that method" ), ) for key, variable in (("max_len", "LAYA_MAX_LEN"), ("head_max_len", "LAYA_HEAD_MAX_LEN")): diff --git a/s1a/desktop/env.py b/s1a/desktop/env.py index f9fbd16..6328a92 100644 --- a/s1a/desktop/env.py +++ b/s1a/desktop/env.py @@ -26,7 +26,7 @@ def clickable(element: Element) -> bool: class WindowEnv: """The mechanics: bind to the app's window, offer its clickable elements, click one, re-read the window. - ``goal`` rides in every observation for the model in the slot. ``done_when`` reads a snapshot and says whether the task is finished; it is also the score. Without ``execute`` + ``goal`` rides in every observation for the model. ``done_when`` reads a snapshot and says whether the task is finished; it is also the score. Without ``execute`` the first chosen click is recorded as ``planned`` and the episode ends, so a dry run shows one decision and acts on nothing. ``clear_labels`` names a button pressed on ``reset`` when the window has one (a calculator's All Clear). """ diff --git a/s1a/jobs.py b/s1a/jobs.py index abd690b..4b1a52d 100644 --- a/s1a/jobs.py +++ b/s1a/jobs.py @@ -42,7 +42,7 @@ class Episode: jev_input_tokens: int invalid_keys: int cost_usd: float | None - error: str | None = None # why the model in the slot could not play the episode; None for a played one + error: str | None = None # why the model could not play the episode; None for a played one decisions: list[dict[str, Any]] = field(default_factory=list) extra: dict[str, Any] = field(default_factory=dict) views: list[dict[str, Any]] = field(default_factory=list) # views[i] is what act i chose from; the last is final diff --git a/s1a/mcp_server.py b/s1a/mcp_server.py index 0beaa64..d6f60dd 100644 --- a/s1a/mcp_server.py +++ b/s1a/mcp_server.py @@ -23,8 +23,8 @@ "S1A agents: System 1 decision models (TypeSafe Jev, Laya, Cua-S1) in the model slot of openJiuwen agents. Use run_agent for a page task with enumerable " "controls, a game or a quiz, and decide for one selection over options you enumerate. Not for arithmetic, " "constraint puzzles or free-text generation. list_agents gives every agent's flags: a browser agent takes " - "--slot jev --goal '...' and needs a chat-model key (OPENAI_API_KEY or LLM_API_KEY plus MODEL_NAME), a Jev key " - "(TYPESAFE_API_KEY or OPENROUTER_API_KEY) and Node for @playwright/mcp; a tool agent takes --slot, --rethink and " + "--model jev --goal '...' and needs a chat-model key (OPENAI_API_KEY or LLM_API_KEY plus MODEL_NAME), a Jev key " + "(TYPESAFE_API_KEY or OPENROUTER_API_KEY) and Node for @playwright/mcp; a tool agent takes --model, --rethink and " "--episodes; a rail takes --labelled-set. Runs go one at a time per server." ) @@ -74,7 +74,7 @@ def _agent_rows() -> list[dict[str, Any]]: @server.tool() async def run_agent(name: str, flags: list[str]) -> dict[str, Any]: - """Run one agent with the flags `s1a run ` takes, e.g. ["--slot", "jev", "--goal", "..."]. + """Run one agent with the flags `s1a run ` takes, e.g. ["--model", "jev", "--goal", "..."]. A browser agent returns its answer, a tool agent its series summary with the job folder it wrote, a rail its evaluation summary. diff --git a/s1a/rails.py b/s1a/rails.py index fd929b8..b8941da 100644 --- a/s1a/rails.py +++ b/s1a/rails.py @@ -29,7 +29,7 @@ from s1a.spec import Json, RailSpec, Verdict QUESTION = "check" -RAIL_SLOTS = ("jev", "laya") +RAIL_MODEL_NAMES = ("jev", "laya") def question(spec: RailSpec) -> Question: @@ -171,14 +171,14 @@ def parser(spec: RailSpec) -> argparse.ArgumentParser: help="JSONL records with a state and a boolean label; the spec's set when it names one", ) build.add_argument( - "--slot", choices=RAIL_SLOTS, default="jev", help="who answers the question: jev, or laya in process" + "--model", choices=RAIL_MODEL_NAMES, default="jev", help="who answers the question: jev, or laya in process" ) return build async def play(spec: RailSpec, args: argparse.Namespace, *, results_dir: Path) -> dict[str, Any]: """Evaluate the rail on its labelled set with a fresh model; the summary names its job folder.""" - decision_model = build_model(args.slot) + decision_model = build_model(args.model) try: await decision_model.warm() return await evaluate(spec, args.labelled_set, decision_model=decision_model, results_dir=results_dir) diff --git a/s1a/tool/__init__.py b/s1a/tool/__init__.py index c9ca52f..f59cb5d 100644 --- a/s1a/tool/__init__.py +++ b/s1a/tool/__init__.py @@ -1,2 +1,2 @@ # coding: utf-8 -"""The tool front: a DeepAgent that plays through observe and act, with Jev, a chat model, a rule or chance in the slot.""" +"""The tool front: a DeepAgent that plays through observe and act, with Jev, a chat model, a rule or chance in the model slot.""" diff --git a/s1a/tool/loop.py b/s1a/tool/loop.py index 8ec0790..2d71ab1 100644 --- a/s1a/tool/loop.py +++ b/s1a/tool/loop.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""The tool-front loop: jiuwen's DeepAgent over two tools per environment, with a decision model or a chat model in the slot.""" +"""The tool-front loop: jiuwen's DeepAgent over two tools per environment, with a decision model or a chat model in the model slot.""" from __future__ import annotations @@ -26,7 +26,7 @@ from s1a.tool.rethink import RethinkRail from s1a.tool.models import ACT_TOOL, OBSERVE_TOOL, EvalState, ToolDecisionModel -SLOTS = ("jev", "llm", "random", "rule", "laya", "cua") +MODEL_NAMES = ("jev", "llm", "random", "rule", "laya", "cua") EVAL_PROMPT = ( "You play a game through two tools. Call observe first. Then call act with exactly one of the candidate keys the " "last tool result offered, one act per turn, until done is true. Then reply with one line: the final score." @@ -34,7 +34,7 @@ REPEAT_AFTER = 3 GIVE_UP_AFTER = 3 LLM_ITERATION_HEADROOM = 2 # the chat model spends turns on malformed or unknown keys; its cap is this many budgets -SLOT_ITERATION_HEADROOM = 2 # observe and the final turn, beyond the act budget +ITERATION_HEADROOM = 2 # observe and the final turn, beyond the act budget WORKSPACE = HOME / "runs" / "evals" # the DeepAgent scaffolds SOUL.md, memory/ and friends here, not in the repo root @@ -110,7 +110,7 @@ async def invoke(self, inputs: dict[str, Any], **kwargs: Any) -> str: try: await self._env.step(key) except Exception as exc: - state.error = f"act failed: {exc}" # the harness feeds the raise back to the model; every slot then stops + state.error = f"act failed: {exc}" # the harness feeds the raise back to the model; every model then stops state.act_calls.append({"key": key, "accepted": False}) raise state.act_calls.append({"key": key, "accepted": True}) @@ -128,7 +128,7 @@ def build_env_tools(env: Env, state: EvalState, *, agent_name: str) -> list[Tool def build_slot_model( - slot: str, + model_name: str, env: Env, state: EvalState, *, @@ -136,18 +136,18 @@ def build_slot_model( chat: Model | None, decision_model: DecisionModel | None, ) -> Model: - """The chat model for the llm slot; the one slot model over the decision model for every other slot.""" - match slot: + """The chat model for ``llm``; the one slot model over the decision model for every other name.""" + match model_name: case "llm": if chat is None: - raise RuntimeError("the llm slot needs the chat model: OPENAI_API_KEY or LLM_API_KEY, and MODEL_NAME") + raise RuntimeError("--model llm needs the chat model: OPENAI_API_KEY or LLM_API_KEY, and MODEL_NAME") return chat case "jev" | "laya" | "cua" | "random" | "rule": if decision_model is None: - raise RuntimeError(f"the {slot} slot needs a decision model") + raise RuntimeError(f"--model {model_name} needs a decision model") return ToolDecisionModel(env, state, rules=rules, decision_model=decision_model, fallback=chat) case _: - raise ValueError(f"unknown slot {slot!r}; one of {SLOTS}") + raise ValueError(f"unknown model {model_name!r}; one of {MODEL_NAMES}") def create_eval_agent( @@ -216,7 +216,7 @@ async def run_episode( spec: ToolAgentSpec, env: Env, *, - slot: str, + model_name: str, seed: int, chat: Model | None, decision_model: DecisionModel | None, @@ -228,15 +228,15 @@ async def run_episode( ) -> Episode: """One episode through the agent: reset, one conversation, the ticks, rethinks, tokens and dollars into the Episode. - ``max_acts`` bounds the acts for every slot; ``timeout_s`` bounds the wall clock, and a timed-out episode + ``max_acts`` bounds the acts for every model; ``timeout_s`` bounds the wall clock, and a timed-out episode keeps its score so far with ``result_type: timeout``.""" state = EvalState(max_acts=max_acts) await env.reset() first_view = await view_of(env, state) counted = CountingModel(chat, state.chat) if chat is not None else None - model = build_slot_model(slot, env, state, rules=spec.rules, chat=counted, decision_model=decision_model) + model = build_slot_model(model_name, env, state, rules=spec.rules, chat=counted, decision_model=decision_model) rail = None - if rethink_on and spec.budget.stall_after > 0 and slot != "llm": # the chat model reads no plan or block + if rethink_on and spec.budget.stall_after > 0 and model_name != "llm": # the chat model reads no plan or block rail = RethinkRail( state, rules=spec.rules, @@ -246,7 +246,7 @@ async def run_episode( repeat_after=REPEAT_AFTER, give_up_after=GIVE_UP_AFTER, ) - cap = (max_acts + SLOT_ITERATION_HEADROOM) * (LLM_ITERATION_HEADROOM if slot == "llm" else 1) + cap = (max_acts + ITERATION_HEADROOM) * (LLM_ITERATION_HEADROOM if model_name == "llm" else 1) agent = create_eval_agent(spec, env, model, state, rethink=rail, max_iterations=cap) started_at = now_iso() started = time.perf_counter() @@ -262,7 +262,7 @@ async def run_episode( await agent.cleanup_task_resources() agent.ability_manager.teardown_tools() # the act and observe cards hold the env and the state await Runner.release(conversation_id) - if slot == "llm": + if model_name == "llm": state.ticks = llm_decisions(state.chat, state.act_calls) if log: for tick in state.ticks: diff --git a/s1a/tool/models.py b/s1a/tool/models.py index d02172b..1e76e33 100644 --- a/s1a/tool/models.py +++ b/s1a/tool/models.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""The model in the eval loop's slot: each turn is one ``act(key)`` tool call chosen by the model behind it. +"""The model in the eval loop's model slot: each turn is one ``act(key)`` tool call chosen by the model behind it. jiuwen's DeepAgent calls ``invoke`` or ``stream`` with the turn's tool list. A turn whose tools include ``act`` is a decision turn: the model reads the environment directly and answers with exactly one ``act`` call. Any @@ -25,13 +25,13 @@ def placeholder_model() -> Model: - """A config the harness accepts for a slot that never makes a chat call (``Model`` refuses a missing one).""" + """A config the harness accepts for a model that never makes a chat call (``Model`` refuses a missing one).""" return init_model(provider="openai", model_name="none", api_key="unused", api_base="https://api.openai.com/v1") @dataclass class EvalState: - """Per-episode facts shared by the model in the slot, the ``act`` tool and the rethink rail.""" + """Per-episode facts shared by the slot model, the ``act`` tool and the rethink rail.""" plan: str = "" notices: list[str] = field(default_factory=list) diff --git a/s1a/tool/series.py b/s1a/tool/series.py index 6b79759..17aed74 100644 --- a/s1a/tool/series.py +++ b/s1a/tool/series.py @@ -12,17 +12,17 @@ from s1a.jobs import Episode, summarize, write_job from s1a.pricing import chat_prices, cost_usd, env_prices from s1a.spec import Json, ToolAgentSpec, positive_float, positive_int -from s1a.tool.loop import SLOTS, run_episode +from s1a.tool.loop import MODEL_NAMES, run_episode def parser(spec: ToolAgentSpec) -> argparse.ArgumentParser: """The shared flags, with the spec's budget as their defaults, then the agent's own flags.""" build = argparse.ArgumentParser(prog=f"s1a run {spec.name}", description=spec.description) build.add_argument( - "--slot", - choices=SLOTS, + "--model", + choices=MODEL_NAMES, required=True, - help="who decides: jev, the chat model, chance, the rule baseline, or laya and cua in process", + help="who decides: jev (over HTTP), laya or cua (in process), llm (the chat model in MODEL_NAME), random, or rule (the agent's baseline)", ) build.add_argument( "--rethink", @@ -55,7 +55,7 @@ def parser(spec: ToolAgentSpec) -> argparse.ArgumentParser: def price_episodes(episodes: list[Episode]) -> None: - """Dollars for the episodes that spent chat tokens: one catalogue lookup, and none when no slot called the chat model.""" + """Dollars for the episodes that spent chat tokens: one catalogue lookup, and none when nothing called the chat model.""" spent = [e for e in episodes if e.chat_input_tokens + e.chat_output_tokens] if not spent: return @@ -79,19 +79,19 @@ async def play(spec: ToolAgentSpec, args: argparse.Namespace, *, results_dir: Pa its job folder, then raises.""" env_prices() chat = optional_chat_model() - if args.slot == "llm" and chat is None: - raise RuntimeError("the llm slot needs the chat model: OPENAI_API_KEY or LLM_API_KEY, and MODEL_NAME") + if args.model == "llm" and chat is None: + raise RuntimeError("--model llm needs the chat model: OPENAI_API_KEY or LLM_API_KEY, and MODEL_NAME") if args.rethink == "on" and spec.budget.stall_after > 0 and chat is None: raise RuntimeError("--rethink on needs the chat model for plans: OPENAI_API_KEY or LLM_API_KEY, and MODEL_NAME") - shared = build_model(args.slot) if args.slot in ("jev", "laya", "cua") else None + shared = build_model(args.model) if args.model in ("jev", "laya", "cua") else None run = await asyncio.to_thread(spec.series, args) # question fetches, game file parsing: seconds of blocking I/O - if args.slot == "rule": + if args.model == "rule": shared = build_model("rule", rule=run.baseline) def model_for(seed: int) -> DecisionModel | None: - if args.slot == "llm": + if args.model == "llm": return None - if args.slot == "random": + if args.model == "random": return build_model("random", seed=seed) return shared @@ -105,7 +105,7 @@ def model_for(seed: int) -> DecisionModel | None: episode = await run_episode( spec, env, - slot=args.slot, + model_name=args.model, seed=seed, chat=chat, decision_model=model_for(seed), diff --git a/scripts/browser_showcase.sh b/scripts/browser_showcase.sh index c4ec0d7..34adfbd 100755 --- a/scripts/browser_showcase.sh +++ b/scripts/browser_showcase.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# One browser agent on both slots, headed, with a frame after every browser call from the run's own Playwright MCP -# session (evals/replay/cast.py), then the pair GIFs from the median run of each slot: replay.gif (the replay page) +# One browser agent on both models, headed, with a frame after every browser call from the run's own Playwright MCP +# session (evals/replay/cast.py), then the pair GIFs from the median run of each model: replay.gif (the replay page) # and strip.gif (the raw frames under a header band). # -# scripts/browser_showcase.sh allrecipes [RUNS] # RUNS runs per slot, default 1; ROOT= renders an existing root with RUNS=0 +# scripts/browser_showcase.sh allrecipes [RUNS] # RUNS runs per model, default 1; ROOT= renders an existing root with RUNS=0 # PYTHON=/path/to/python scripts/browser_showcase.sh ... # an interpreter with the report extra # # Needs the Jev key and the chat-model key in .env, Node for the MCP server, and a screen: the sites these agents @@ -20,22 +20,22 @@ speed=${SPEED:-8} eval "$(scripts/pw_mcp_nosettle.sh)" # the MCP copy without the settle sleeps: PLAYWRIGHT_MCP_COMMAND and _ARGS server="$PLAYWRIGHT_MCP_COMMAND $PLAYWRIGHT_MCP_ARGS" for ((i = 1; i <= runs; i++)); do # not seq: BSD seq counts down from 1 to 0 - for slot in jev llm; do - logs=$root/$slot-$i + for model in jev llm; do + logs=$root/$model-$i mkdir -p "$logs" - batch=off; [ "$slot" = jev ] && batch=on - echo "=== $agent $slot run $i -> $logs" >&2 + batch=off; [ "$model" = jev ] && batch=on + echo "=== $agent $model run $i -> $logs" >&2 PLAYWRIGHT_MCP_COMMAND=$PYTHON PLAYWRIGHT_MCP_ARGS="-m evals.replay.cast --frames $logs/frames -- $server" \ - "$PYTHON" -m s1a run "$agent" --slot "$slot" --batch "$batch" --headed --logs-dir "$logs" > "$logs/stdout.json" + "$PYTHON" -m s1a run "$agent" --model "$model" --batch "$batch" --headed --logs-dir "$logs" > "$logs/stdout.json" done done -median() { # the slot's run with the median wall clock; the lower middle for an even count +median() { # the model's run with the median wall clock; the lower middle for an even count "$PYTHON" - "$root" "$1" <<'PY' import json, sys from pathlib import Path -root, slot = Path(sys.argv[1]), sys.argv[2] -runs = sorted(root.glob(f"{slot}-*"), key=lambda d: json.loads((d / "answer.json").read_text())["elapsed_ms"]) +root, model = Path(sys.argv[1]), sys.argv[2] +runs = sorted(root.glob(f"{model}-*"), key=lambda d: json.loads((d / "answer.json").read_text())["elapsed_ms"]) print(runs[(len(runs) - 1) // 2]) PY } diff --git a/scripts/showcase.sh b/scripts/showcase.sh index f70abc5..fb0d9a4 100755 --- a/scripts/showcase.sh +++ b/scripts/showcase.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# One showcase episode per eval and slot, same seed for both slots, headless, frames on for the browser games; +# One showcase episode per eval and model, same seed for both models, headless, frames on for the browser games; # then the pair page under evals/showcase/replays// (with its frames) and the GIF alone under # docs/results//showcase/replay.gif. Never feeds the matrix: every job lands under evals/showcase/, which # evals.table does not read. @@ -15,10 +15,10 @@ THOR_PYTHON=${THOR_PYTHON:-$PYTHON} seed=${1:-0}; shift || true evals=("$@"); [ ${#evals[@]} -eq 0 ] && evals=(blackjack game2048 millionaire alfworld) -run() { # eval, slot, extra args...: prints the trial folder; the run's last stdout line is its summary JSON - local eval=$1 slot=$2; shift 2 +run() { # eval, model, extra args...: prints the trial folder; the run's last stdout line is its summary JSON + local eval=$1 model=$2; shift 2 local job - job=$("$PYTHON" -m s1a run "$eval" --slot "$slot" --episodes 1 --seed "$seed" --showcase "$@" | tail -1 \ + job=$("$PYTHON" -m s1a run "$eval" --model "$model" --episodes 1 --seed "$seed" --showcase "$@" | tail -1 \ | "$PYTHON" -c 'import json, sys; print(json.load(sys.stdin)["job_dir"])') ls -d "$job"/*--*/ | head -1 } diff --git a/skills/s1a/SKILL.md b/skills/s1a/SKILL.md index 6c77e5c..2fa1894 100644 --- a/skills/s1a/SKILL.md +++ b/skills/s1a/SKILL.md @@ -26,29 +26,29 @@ generation, or a task that needs a value the page never shows. ```bash s1a list -s1a run flights --slot jev --goal "" -s1a run --slot jev --rethink off --episodes 3 +s1a run flights --model jev --goal "" +s1a run --model jev --rethink off --episodes 3 s1a decide --state '' --option key="what it means" --option other="what it means" --rules "" s1a decide --state '{"title": "Charged twice", "description": "I was charged twice for order 4411 and I want the second charge refunded.", "order_status": "delivered"}' \ --option logistics="delivery tracking, delivery progress or delivery problems" --option payment="charges, failed payments or duplicate payments" \ --option returns="requests for returns, exchanges or refunds" --option account="login or account access problems" \ --option human="insufficient information, several independent requests, or an explicit request for a human" \ --rules "Route the current explicit request to exactly one queue. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human." -s1a run ticket_router --slot jev --rethink off --episodes 1 # the shipped router: 30 labelled tickets, five queues, scores the correct routes +s1a run ticket_router --model jev --rethink off --episodes 1 # the shipped router: 30 labelled tickets, five queues, scores the correct routes ``` Every `run` prints one JSON object on stdout and nothing else there. The harness logs go to files under the -checkout's `runs/logs`. A browser agent's object has `ok`, `final`, `error` and `usage` on both slots; `final` is -the answer. With `--slot jev` it also has `report` and, when present (absent on a timeout), `status` and -`terminal`: `terminal.url` and `terminal.title` are where the answer was read. With `--slot llm` it has `browser_result`, the subagent's own +checkout's `runs/logs`. A browser agent's object has `ok`, `final`, `error` and `usage` with both models; `final` is +the answer. With `--model jev` it also has `report` and, when present (absent on a timeout), `status` and +`terminal`: `terminal.url` and `terminal.title` are where the answer was read. With `--model llm` it has `browser_result`, the subagent's own verdict; `status`, `report` and `terminal` are absent there. A task takes seconds to a few minutes; `--timeout` sets the wall clock (the agent's own default, 180 s for `flights`) and `--headed` shows the browser. A tool agent's object is the series summary with `job_dir`, the job folder it wrote. `decide` prints `choice`, `probabilities`, `confidence` and `ms`. Exit codes: 0 for a finished run, including one whose JSON has `ok: false` and an `error`; 1 for a run, key or model error, one line on stderr; 2 for a usage error (an unknown agent, bad flags, a malformed `--state`, `--option` or `@file`). -`--slot jev` is TypeSafe Jev; `--slot laya` is Laya, an open-weight System 1 decision model that runs in process after -`uv sync --extra laya`, with the same outputs and no Jev key; `--slot cua` is Cua-S1 Nano, a small option scorer in +`--model jev` is TypeSafe Jev; `--model laya` is Laya, an open-weight System 1 decision model that runs in process after +`uv sync --extra laya`, with the same outputs and no Jev key; `--model cua` is Cua-S1 Nano, a small option scorer in process after `uv sync --extra cua`, a baseline; tool agents also take `llm`, `random` and `rule`. ## Keys diff --git a/tests/support.py b/tests/support.py index 191b3a0..ad6aba1 100644 --- a/tests/support.py +++ b/tests/support.py @@ -79,18 +79,18 @@ async def _decide(self, observation: Observation, questions: dict[str, Question] def browser_result(summary: str, *, status: str) -> str: - """The subagent's structured completion around ``summary``, as the llm slot's final text.""" + """The subagent's structured completion around ``summary``, as the chat model's final text.""" return json.dumps( {"browser_result": {"status": status, "terminal_reason": "runtime_completion_validated", "summary": summary}} ) async def browse_offline( - spec: BrowserAgentSpec, policy: BrowserPolicy, *, slot: str, max_steps: int + spec: BrowserAgentSpec, policy: BrowserPolicy, *, model_name: str, max_steps: int ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: """``browse`` with the subagent factory and the browser run faked: the answer, what the factory saw, the files. - No Runner, no browser, no key: a decision-model slot's final text is a DONE summary, the llm slot's a completion. + No Runner, no browser, no key: a decision model's final text is a DONE summary, the chat model's a completion. """ seen: dict[str, Any] = {} @@ -101,7 +101,7 @@ def fake_factory(model: Any, **kwargs: Any) -> Any: async def fake_run(agent: Any, goal: str, *, timeout_s: float) -> dict[str, Any]: seen.update(goal=goal, timeout_s=timeout_s) done = json.dumps({"status": "DONE", "reason": "", "url": "https://x", "answer": "Three flights."}) - final = browser_result("42", status="completed") if slot == "llm" else done + final = browser_result("42", status="completed") if model_name == "llm" else done return {"ok": True, "final": final, "screenshot": None, "error": None} with ( @@ -113,14 +113,14 @@ async def fake_run(agent: Any, goal: str, *, timeout_s: float) -> dict[str, Any] answer = await browse.browse( spec, policy, - slot=slot, + model_name=model_name, goal="Show one-way flights", timeout_s=30, max_steps=max_steps, logs_dir=Path(tmp), headless=True, chat=chat_model_from_env(), - decision_model=None if slot == "llm" else NoDecisionModel(), + decision_model=None if model_name == "llm" else NoDecisionModel(), ) files = sorted(p.name for p in Path(tmp).iterdir()) return answer, seen, files diff --git a/tests/test_agents_alfworld.py b/tests/test_agents_alfworld.py index e2109f3..e852d52 100644 --- a/tests/test_agents_alfworld.py +++ b/tests/test_agents_alfworld.py @@ -53,12 +53,12 @@ async def test_the_oracle_stops_when_the_expert_falls_back_to_look(self) -> None rule({}, await env.candidates()) async def test_more_episodes_than_games_is_an_error(self) -> None: - flags = Namespace(offset=0, stride=1, episodes=3, max_steps=50, slot="rule") + flags = Namespace(offset=0, stride=1, episodes=3, max_steps=50, model_name="rule") with patch("s1a.agents.alfworld.solvable_game_files", return_value=["a/game.tw-pddl", "b/game.tw-pddl"]): with self.assertRaisesRegex(ValueError, "--episodes 3 but only 2 games"): make_series(flags) - async def test_the_decision_model_slots_see_no_look_or_examine(self) -> None: + async def test_the_decision_models_see_no_look_or_examine(self) -> None: env = AlfworldEnv([solvable_game_files()[0]], 50, every_command=False) await env.reset() self.assertFalse([c for c in await env.candidates() if c.startswith(("look", "examine", "inventory"))]) diff --git a/tests/test_agents_desktop.py b/tests/test_agents_desktop.py index 4796afe..ba6a20c 100644 --- a/tests/test_agents_desktop.py +++ b/tests/test_agents_desktop.py @@ -57,15 +57,15 @@ def test_the_plan_rule_follows_the_labels_with_variants_then_says_done_or_abstai def test_the_flags_require_app_goal_and_expect_and_default_to_a_dry_run(self) -> None: args = series.parser(desktop.SPEC).parse_args( - ["--slot", "jev", "--rethink", "off", "--episodes", "1", *CALCULATOR] + ["--model", "jev", "--rethink", "off", "--episodes", "1", *CALCULATOR] ) self.assertEqual((args.execute, args.plan, args.clear), (False, "", "")) with self.assertRaises(SystemExit): - series.parser(desktop.SPEC).parse_args(["--slot", "jev", "--rethink", "off", "--episodes", "1"]) + series.parser(desktop.SPEC).parse_args(["--model", "jev", "--rethink", "off", "--episodes", "1"]) def test_without_a_plan_there_is_no_rule_baseline(self) -> None: args = series.parser(desktop.SPEC).parse_args( - ["--slot", "rule", "--rethink", "off", "--episodes", "1", *CALCULATOR] + ["--model", "rule", "--rethink", "off", "--episodes", "1", *CALCULATOR] ) with patch.object(desktop, "driver_from_env", lambda label: FakeCalculator()): self.assertIsNone(desktop.make_series(args).baseline) @@ -121,7 +121,7 @@ async def spawn(*args: str) -> _Exited: class TestThroughTheSeries(IsolatedAsyncioTestCase): async def _play(self, fake: FakeCalculator, *flags: str) -> tuple[dict, dict]: args = series.parser(desktop.SPEC).parse_args( - ["--slot", "rule", "--rethink", "off", "--episodes", "1", *CALCULATOR, *PLAN, *flags] + ["--model", "rule", "--rethink", "off", "--episodes", "1", *CALCULATOR, *PLAN, *flags] ) with ( tempfile.TemporaryDirectory() as tmp, diff --git a/tests/test_agents_ticket_router.py b/tests/test_agents_ticket_router.py index c79e67d..20180a4 100644 --- a/tests/test_agents_ticket_router.py +++ b/tests/test_agents_ticket_router.py @@ -194,7 +194,7 @@ def test_keywords_baseline_handles_unique_matches_and_ambiguity(self): class TestTicketRouterThroughTheLoop(IsolatedAsyncioTestCase): - async def _play(self, slot, max_steps=5): + async def _play(self, model_name, max_steps=5): module = router() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -202,8 +202,8 @@ async def _play(self, slot, max_steps=5): data.write_text("\n".join(json.dumps(row) for row in TICKETS), encoding="utf-8") args = series.parser(module.SPEC).parse_args( [ - "--slot", - slot, + "--model", + model_name, "--rethink", "off", "--episodes", @@ -220,8 +220,8 @@ async def _play(self, slot, max_steps=5): ): async with started_runner(): result = await series.play(module.SPEC, args, results_dir=root / "results") - # Filesystem enumeration has no guaranteed order; exercise different orders for the two slots. - paths = sorted(Path(result["job_dir"]).glob("*/agent/episode.json"), reverse=slot == "random") + # Filesystem enumeration has no guaranteed order; exercise different orders for the two models. + paths = sorted(Path(result["job_dir"]).glob("*/agent/episode.json"), reverse=model_name == "random") episodes = [json.loads(p.read_text(encoding="utf-8")) for p in paths] return result, episodes @@ -258,6 +258,6 @@ async def test_step_budget_is_reported_as_partial_not_full_success(self): def test_rethink_on_is_rejected_for_independent_tickets(self): module = router() - flags = series.parser(module.SPEC).parse_args(["--slot", "rule", "--rethink", "on", "--episodes", "1"]) + flags = series.parser(module.SPEC).parse_args(["--model", "rule", "--rethink", "on", "--episodes", "1"]) with self.assertRaises(ValueError): module.make_series(flags) diff --git a/tests/test_browse.py b/tests/test_browse.py index ae55f31..71f9c20 100644 --- a/tests/test_browse.py +++ b/tests/test_browse.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""The browser front: the answer shape, the assembly of the subagent per slot.""" +"""The browser front: the answer shape, the assembly of the subagent per model.""" from __future__ import annotations @@ -28,7 +28,7 @@ from support import NoDecisionModel, browse_offline, browser_result SPEC = flights.SPEC -GOAL = ["--slot", "jev", "--goal", "x"] +GOAL = ["--model", "jev", "--goal", "x"] class TestParser(TestCase): @@ -55,12 +55,12 @@ def test_a_timeout_or_max_steps_at_or_below_zero_is_a_usage_error(self) -> None: browse.parser(SPEC).parse_args([*GOAL, *flags]) self.assertEqual(caught.exception.code, 2) - def test_the_slot_takes_a_model_or_the_chat_model(self) -> None: - self.assertEqual(browse.BROWSER_SLOTS, ("jev", "laya", "cua", "llm")) - for slot in browse.BROWSER_SLOTS: - self.assertEqual(browse.parser(SPEC).parse_args(["--slot", slot, "--goal", "x"]).slot, slot) + def test_the_model_flag_takes_a_decision_model_or_the_chat_model(self) -> None: + self.assertEqual(browse.BROWSER_MODEL_NAMES, ("jev", "laya", "cua", "llm")) + for model_name in browse.BROWSER_MODEL_NAMES: + self.assertEqual(browse.parser(SPEC).parse_args(["--model", model_name, "--goal", "x"]).model, model_name) with self.assertRaises(SystemExit): - browse.parser(SPEC).parse_args(["--slot", "random", "--goal", "x"]) + browse.parser(SPEC).parse_args(["--model", "random", "--goal", "x"]) class TestSolverContract(TestCase): @@ -75,7 +75,7 @@ def test_terminal_summary_survives_text_the_rail_appends(self) -> None: + "\n\nThe runtime could not verify completion." ) self.assertEqual(browse.terminal_summary(final)["answer"], "42") - self.assertEqual(browse.finish_decision_model(_answer(final), slot="jev")["final"], "42") + self.assertEqual(browse.finish_decision_model(_answer(final), model_name="jev")["final"], "42") def test_terminal_summary_reads_the_summary_inside_browser_result(self) -> None: final = _completed(json.dumps({"status": "BLOCKED", "reason": "", "page_text": "Flights"})) @@ -107,7 +107,7 @@ def _answer(final: str) -> dict[str, Any]: class TestFinishJev(TestCase): def test_blocked_is_a_failure_even_when_the_subagent_says_completed(self) -> None: final = _completed(json.dumps({"status": "BLOCKED", "reason": "", "page_text": "Flights", "answer": ""})) - answer = browse.finish_decision_model(_answer(final), slot="laya") + answer = browse.finish_decision_model(_answer(final), model_name="laya") self.assertEqual((answer["ok"], answer["final"], answer["status"]), (False, "", "BLOCKED")) self.assertEqual(answer["error"], "laya BLOCKED: no reason given") @@ -123,13 +123,13 @@ def test_done_takes_the_policy_answer_and_clears_the_harness_verdict(self) -> No "ok": False, "error": "result_type='error': partial", } - answer = browse.finish_decision_model(flagged, slot="jev") + answer = browse.finish_decision_model(flagged, model_name="jev") self.assertEqual((answer["ok"], answer["final"], answer["error"]), (True, "Three flights from 412 USD.", None)) self.assertEqual(answer["terminal"], summary) def test_blocked_after_progress_keeps_the_pages_answer(self) -> None: summary = {"status": "BLOCKED", "reason": "oscillating between two pages", "answer": "Rated 4.6 by 243."} - answer = browse.finish_decision_model(_answer(json.dumps(summary)), slot="jev") + answer = browse.finish_decision_model(_answer(json.dumps(summary)), model_name="jev") self.assertEqual( (answer["ok"], answer["final"], answer["status"], answer["error"]), (True, "Rated 4.6 by 243.", "BLOCKED", None), @@ -137,21 +137,21 @@ def test_blocked_after_progress_keeps_the_pages_answer(self) -> None: def test_done_without_an_answer_is_a_failure(self) -> None: final = _completed(json.dumps({"status": "DONE", "reason": "", "page_text": "x", "answer": ""})) - answer = browse.finish_decision_model(_answer(final), slot="cua") + answer = browse.finish_decision_model(_answer(final), model_name="cua") self.assertEqual((answer["ok"], answer["error"]), (False, "cua DONE without an answer")) class TestBrowseAssembly(IsolatedAsyncioTestCase): """``browse`` builds the real slot model from the spec; the factory and the browser run are faked, no Runner runs.""" - async def _browse(self, slot: str, *, batch: bool) -> tuple[dict[str, Any], dict[str, Any], list[str]]: + async def _browse(self, model_name: str, *, batch: bool) -> tuple[dict[str, Any], dict[str, Any], list[str]]: policy = BrowserPolicy(prefetch_values=True, batch_actions=batch, goal_value_cache=False) - return await browse_offline(SPEC, policy, slot=slot, max_steps=7) + return await browse_offline(SPEC, policy, model_name=model_name, max_steps=7) - async def test_a_decision_model_slot_puts_the_slot_model_in_the_slot_and_records_its_ticks(self) -> None: - for slot in ("jev", "laya"): - with self.subTest(slot=slot): - answer, seen, files = await self._browse(slot, batch=False) + async def test_a_decision_model_puts_the_slot_model_in_the_slot_and_records_its_ticks(self) -> None: + for model_name in ("jev", "laya"): + with self.subTest(model_name=model_name): + answer, seen, files = await self._browse(model_name, batch=False) self.assertIsInstance(seen["model"], BrowserDecisionModel) self.assertIsInstance(seen["model"]._decision_model, NoDecisionModel) self.assertEqual( @@ -168,7 +168,7 @@ async def test_batched_actions_ask_for_the_unsafe_dev_capability(self) -> None: _answer, seen, _files = await self._browse("jev", batch=True) self.assertEqual(seen["browser_capabilities"], ["unsafe_dev"]) - async def test_llm_slot_keeps_the_counting_model_in_the_slot(self) -> None: + async def test_llm_keeps_the_counting_model_in_the_slot(self) -> None: answer, seen, files = await self._browse("llm", batch=False) self.assertIsInstance(seen["model"], CountingModel) self.assertIs(_browser_model_with_temperature(seen["model"], DEFAULT_BROWSER_AGENT_TEMPERATURE), seen["model"]) @@ -176,15 +176,19 @@ async def test_llm_slot_keeps_the_counting_model_in_the_slot(self) -> None: self.assertEqual(answer["usage"]["chat_temperature"], 0.0, "the sampling setting the baseline really ran at") self.assertEqual(files, ["chat_calls.json"]) - async def test_a_model_slot_refuses_to_run_without_a_model(self) -> None: + async def test_jev_and_laya_refuse_to_run_without_a_decision_model(self) -> None: policy = BrowserPolicy(prefetch_values=True, batch_actions=False, goal_value_cache=False) - for slot in ("jev", "laya"): - with self.subTest(slot=slot), patch.dict(os.environ, ENV, clear=True), tempfile.TemporaryDirectory() as tmp: + for model_name in ("jev", "laya"): + with ( + self.subTest(model_name=model_name), + patch.dict(os.environ, ENV, clear=True), + tempfile.TemporaryDirectory() as tmp, + ): with self.assertRaises(RuntimeError) as caught: await browse.browse( SPEC, policy, - slot=slot, + model_name=model_name, goal="g", timeout_s=1, max_steps=1, @@ -193,7 +197,7 @@ async def test_a_model_slot_refuses_to_run_without_a_model(self) -> None: chat=chat_model_from_env(), decision_model=None, ) - self.assertIn(slot, str(caught.exception)) + self.assertIn(model_name, str(caught.exception)) class _FakeAgent: @@ -242,7 +246,7 @@ async def test_a_timed_out_run_is_released_too(self) -> None: class TestPlay(IsolatedAsyncioTestCase): - """``play`` builds the slot's decision_model, hands the flags to ``browse`` and returns the answer without the ticks; + """``play`` builds the decision model ``--model`` names, hands the flags to ``browse`` and returns the answer without the ticks; no profiler without a path.""" async def test_the_answer_comes_back_without_ticks_and_without_a_profile(self) -> None: @@ -253,19 +257,19 @@ async def fake_browse(spec: Any, policy: Any, **kwargs: Any) -> dict[str, Any]: return {"ok": True, "final": "42", "error": None, "ticks": [{"tick": 1}], "report": {}, "usage": {}} with tempfile.TemporaryDirectory() as tmp, patch.dict(os.environ, ENV, clear=True): - args = browse.parser(SPEC).parse_args(["--slot", "llm", "--goal", "g", "--logs-dir", tmp, "--batch", "on"]) + args = browse.parser(SPEC).parse_args(["--model", "llm", "--goal", "g", "--logs-dir", tmp, "--batch", "on"]) with patch.object(browse, "browse", fake_browse), patch.object(browse, "BrowserProfiler", None): answer = await browse.play(SPEC, args) written = json.loads((Path(tmp) / "answer.json").read_text(encoding="utf-8")) self.assertEqual(answer, {"ok": True, "final": "42", "error": None, "report": {}, "usage": {}}) self.assertEqual( - written, {**answer, "agent": "flights", "slot": "llm"}, "the result lands next to the run's records" + written, {**answer, "agent": "flights", "model": "llm"}, "the result lands next to the run's records" ) - self.assertEqual((seen["slot"], seen["max_steps"], seen["policy"].batch_actions), ("llm", 100, True)) + self.assertEqual((seen["model_name"], seen["max_steps"], seen["policy"].batch_actions), ("llm", 100, True)) self.assertEqual((seen["goal"], seen["timeout_s"], seen["headless"]), ("g", 180.0, True)) self.assertIsNone(seen["decision_model"], "the chat model needs no model") - async def test_a_model_slot_builds_its_model_from_the_environment_and_closes_it(self) -> None: + async def test_a_decision_model_is_built_from_the_environment_and_closed(self) -> None: seen: dict[str, Any] = {} closed: list[str] = [] @@ -277,15 +281,15 @@ async def close(self: Any) -> None: closed.append(self.name) with tempfile.TemporaryDirectory() as tmp, patch.dict(os.environ, ENV, clear=True): - args = browse.parser(SPEC).parse_args(["--slot", "jev", "--goal", "g", "--logs-dir", tmp]) + args = browse.parser(SPEC).parse_args(["--model", "jev", "--goal", "g", "--logs-dir", tmp]) with patch.object(browse, "browse", fake_browse), patch.object(JevModel, "close", close): await browse.play(SPEC, args) self.assertIsInstance(seen["decision_model"], JevModel) self.assertEqual(closed, ["jev"], "the model is closed once the task is over") - async def test_the_profiler_needs_a_decision_model_slot(self) -> None: + async def test_the_profiler_needs_a_decision_model(self) -> None: with tempfile.TemporaryDirectory() as tmp, patch.dict(os.environ, ENV, clear=True): - args = browse.parser(SPEC).parse_args(["--slot", "llm", "--goal", "g", "--profile-out", f"{tmp}/p.json"]) + args = browse.parser(SPEC).parse_args(["--model", "llm", "--goal", "g", "--profile-out", f"{tmp}/p.json"]) with patch.object(browse, "browse", None), self.assertRaises(RuntimeError): await browse.play(SPEC, args) diff --git a/tests/test_browser_policy.py b/tests/test_browser_policy.py index be2f529..6217256 100644 --- a/tests/test_browser_policy.py +++ b/tests/test_browser_policy.py @@ -456,7 +456,7 @@ async def test_a_laya_shaped_model_fills_the_slot_the_same_way(self) -> None: message = await slot_model.invoke(_MESSAGES, tools=_TOOLS) self.assertEqual(message.tool_calls[0].name, "browser_click") - self.assertTrue(message.tool_calls[0].id.startswith("scripted-"), "the call id names the slot, not Jev") + self.assertTrue(message.tool_calls[0].id.startswith("scripted-"), "the call id names the slot model, not Jev") ((observation, questions),) = decision_model.calls self.assertEqual(sorted(questions), ["click_target", "operation", "select_target", "type_text_target"]) self.assertEqual(observation.state["page"]["title"], "Flights") diff --git a/tests/test_cli.py b/tests/test_cli.py index 10298d2..729f365 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -25,7 +25,7 @@ from s1a.tool import loop, series from support import COUNTER -RUN = ["run", "counter", "--slot", "random", "--rethink", "off", "--episodes", "1", "--showcase"] +RUN = ["run", "counter", "--model", "random", "--rethink", "off", "--episodes", "1", "--showcase"] NO_KEYS = {"TYPESAFE_API_KEY": "", "OPENROUTER_API_KEY": "", "TYPESAFE_API_URL": ""} HAVE_RLCARD = importlib.util.find_spec("rlcard") is not None # the one tool agent that runs offline @@ -64,7 +64,7 @@ def denied(name: str) -> None: raise PermissionError("[Errno 13] Permission denied: 'results'") with patch.object(agents, "load", denied): - code, out, err = _main(["run", "counter", "--slot", "random", "--rethink", "off", "--episodes", "1"]) + code, out, err = _main(["run", "counter", "--model", "random", "--rethink", "off", "--episodes", "1"]) self.assertEqual((code, out), (1, "")) self.assertIn("Permission denied", err) @@ -73,7 +73,7 @@ def broken(name: str) -> None: raise IndexError("list index out of range") with patch.object(agents, "load", broken), self.assertRaises(IndexError): - _main(["run", "counter", "--slot", "random", "--rethink", "off", "--episodes", "1"]) + _main(["run", "counter", "--model", "random", "--rethink", "off", "--episodes", "1"]) def test_a_tool_agent_runs_offline_and_prints_its_summary_with_the_job_folder(self) -> None: with ( @@ -96,7 +96,7 @@ def test_a_missing_key_is_one_line_on_stderr_and_exit_1(self) -> None: patch.object(agents, "load", lambda name: COUNTER), patch.object(series, "optional_chat_model", lambda: None), ): - code, out, err = _main(["run", "counter", "--slot", "jev", "--rethink", "off", "--episodes", "1"]) + code, out, err = _main(["run", "counter", "--model", "jev", "--rethink", "off", "--episodes", "1"]) self.assertEqual((code, out), (1, "")) self.assertEqual(len(err.strip().splitlines()), 1) self.assertIn("TYPESAFE_API_KEY or OPENROUTER_API_KEY", err) @@ -112,7 +112,7 @@ def test_rethink_on_without_the_chat_model_is_one_line_on_stderr_and_exit_1(self patch.object(agents, "load", lambda name: STALLING), patch.object(series, "optional_chat_model", lambda: None), ): - code, out, err = _main(["run", "counter", "--slot", "random", "--rethink", "on", "--episodes", "1"]) + code, out, err = _main(["run", "counter", "--model", "random", "--rethink", "on", "--episodes", "1"]) self.assertEqual((code, out), (1, "")) self.assertEqual(len(err.strip().splitlines()), 1) self.assertIn("--rethink on needs the chat model", err) @@ -148,8 +148,8 @@ def test_prints_the_validated_answer(self) -> None: transport = ScriptedTransport(choose="stand") built: list[str] = [] - def build(slot: str, **kwargs: Any) -> JevModel: - built.append(slot) + def build(model_name: str, **kwargs: Any) -> JevModel: + built.append(model_name) return JevModel(transport) with patch.object(cli, "build_model", build): @@ -165,7 +165,7 @@ def build(slot: str, **kwargs: Any) -> JevModel: ) self.assertEqual(transport.bodies[0]["state"], {"player_total": 18}) - def test_the_slot_flag_picks_the_model_and_the_model_is_closed(self) -> None: + def test_the_model_flag_picks_the_model_and_the_model_is_closed(self) -> None: closed: list[str] = [] class Closing(JevModel): @@ -174,15 +174,15 @@ async def close(self) -> None: built: list[str] = [] - def build(slot: str, **kwargs: Any) -> JevModel: - built.append(slot) + def build(model_name: str, **kwargs: Any) -> JevModel: + built.append(model_name) return Closing(ScriptedTransport()) with patch.object(cli, "build_model", build): - code, _out, _err = _main([*DECIDE, "--slot", "laya"]) + code, _out, _err = _main([*DECIDE, "--model", "laya"]) self.assertEqual((code, built, closed), (0, ["laya"], ["jev"])) with self.assertRaises(SystemExit): - cli.parser().parse_args([*DECIDE, "--slot", "random"]) + cli.parser().parse_args([*DECIDE, "--model", "random"]) def test_malformed_input_is_one_line_on_stderr_and_exit_2_before_any_key_check(self) -> None: option = ["--option", "a=one", "--option", "b=two", "--rules", "none"] @@ -305,7 +305,7 @@ def test_an_unusable_s1a_home_is_one_line_on_stderr_and_exit_1(self) -> None: @skipUnless(HAVE_RLCARD, "the blackjack extra (rlcard) is not installed") def test_a_blackjack_series_prints_one_json_line(self) -> None: done = subprocess.run( - [sys.executable, "-m", "s1a", "run", "blackjack", "--slot", "rule", *RUN[4:]], + [sys.executable, "-m", "s1a", "run", "blackjack", "--model", "rule", *RUN[4:]], capture_output=True, text=True, timeout=300, diff --git a/tests/test_decision_models_cua.py b/tests/test_decision_models_cua.py index 6f80dee..d146e14 100644 --- a/tests/test_decision_models_cua.py +++ b/tests/test_decision_models_cua.py @@ -175,7 +175,7 @@ def snapshot_download(repo_id: str, allow_patterns: list[str]) -> str: self.assertEqual(decision_model.model, "cua-ai/cua-s1-nano-0.1/text") self.assertEqual((decision_model._context_bytes, decision_model._option_bytes), (256, 96)) - def test_the_factory_builds_it_for_the_cua_slot(self) -> None: + def test_the_factory_builds_it_for_cua(self) -> None: with tempfile.TemporaryDirectory() as tmp: (Path(tmp) / "text").mkdir() with ( diff --git a/tests/test_decision_models_factory.py b/tests/test_decision_models_factory.py index c1d1dbe..d38866e 100644 --- a/tests/test_decision_models_factory.py +++ b/tests/test_decision_models_factory.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""Slot name to decision_model, from the environment: ``build_model`` is the door every front uses.""" +"""Model name to decision_model, from the environment: ``build_model`` is the door every front uses.""" from __future__ import annotations @@ -13,7 +13,7 @@ from openjiuwen.core.common.exception.errors import BaseError from s1a.decision_models import ( - DECISION_MODEL_SLOTS, + DECISION_MODEL_NAMES, JevModel, LayaModel, RandomModel, @@ -25,7 +25,7 @@ class TestBuildModel(TestCase): - def test_every_slot_builds_its_class(self) -> None: + def test_every_name_builds_its_class(self) -> None: with patch.dict(os.environ, KEYS): self.assertIsInstance(build_model("jev"), JevModel) fake_laya = SimpleNamespace( @@ -37,17 +37,17 @@ def test_every_slot_builds_its_class(self) -> None: rule = build_model("rule", rule=("always-inc", lambda state, options: "inc")) self.assertIsInstance(rule, RuleModel) self.assertEqual(rule.name, "always-inc") - self.assertEqual(DECISION_MODEL_SLOTS, ("jev", "laya", "cua", "random", "rule")) + self.assertEqual(DECISION_MODEL_NAMES, ("jev", "laya", "cua", "random", "rule")) def test_the_errors(self) -> None: with self.assertRaises(RuntimeError): build_model("rule") with self.assertRaises(ValueError): build_model("llm") - for slot, module in (("laya", "laya"), ("cua", "cua_s1.nano")): + for model_name, module in (("laya", "laya"), ("cua", "cua_s1.nano")): with patch.dict(sys.modules, {module: None}): with self.assertRaises(BaseError) as caught: - build_model(slot) + build_model(model_name) self.assertEqual(caught.exception.status, StatusCode.MODEL_SERVICE_CONFIG_ERROR) with patch.dict(os.environ, {"TYPESAFE_API_KEY": "", "OPENROUTER_API_KEY": ""}): with self.assertRaises(BaseError): diff --git a/tests/test_evals_table.py b/tests/test_evals_table.py index d5f9e1c..870405d 100644 --- a/tests/test_evals_table.py +++ b/tests/test_evals_table.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""The results table: job folders from write_job, one row per eval and slot.""" +"""The results table: job folders from write_job, one row per eval and model.""" from __future__ import annotations @@ -9,7 +9,7 @@ from unittest import TestCase from s1a.jobs import Episode, write_job -from evals.table import markdown, read_results, rows, slot_label +from evals.table import markdown, read_results, rows, model_label def _episode(seed: int, score: float, cost: float | None) -> Episode: @@ -39,12 +39,12 @@ def _episode(seed: int, score: float, cost: float | None) -> Episode: class TestTable(TestCase): - def test_slot_label_drops_any_suite_prefix(self) -> None: - self.assertEqual(slot_label({"agent_info": {"name": "s1a-evals/jev"}}), "jev") - self.assertEqual(slot_label({"agent_info": {"name": "jiuwen-jev-evals/llm"}}), "llm") - self.assertEqual(slot_label({"agent_info": {"name": "basic"}}), "basic") + def test_model_label_drops_any_suite_prefix(self) -> None: + self.assertEqual(model_label({"agent_info": {"name": "s1a-evals/jev"}}), "jev") + self.assertEqual(model_label({"agent_info": {"name": "jiuwen-jev-evals/llm"}}), "llm") + self.assertEqual(model_label({"agent_info": {"name": "basic"}}), "basic") - def test_rows_merge_job_folders_per_slot(self) -> None: + def test_rows_merge_job_folders_per_model(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) write_job("blackjack", [_episode(0, 1.0, 0.0001), _episode(1, -1.0, 0.0001)], results_dir=root) @@ -52,7 +52,7 @@ def test_rows_merge_job_folders_per_slot(self) -> None: table = rows(read_results(root)) - by_key = {(row["eval"], row["slot"]): row for row in table} + by_key = {(row["eval"], row["model"]): row for row in table} blackjack = by_key[("blackjack", "jev")] self.assertEqual((blackjack["N"], blackjack["mean_score"], blackjack["median_s"]), (3, 0.333, 1.5)) self.assertEqual( @@ -72,7 +72,7 @@ def test_errored_trials_are_counted_and_not_scored(self) -> None: self.assertEqual((row["N"], row["errors"], row["mean_score"], row["mean_cost_usd"]), (1, 2, 1.0, 0.0001)) self.assertIn("| blackjack | jev | 1 | 2 | 1.0 [1.0, 1.0] |", markdown([row])) - def test_a_slot_whose_every_trial_errored_has_no_score(self) -> None: + def test_a_model_whose_every_trial_errored_has_no_score(self) -> None: failed = _episode(0, 0.0, None) failed.error = "decision failed: HTTP 401" with tempfile.TemporaryDirectory() as tmp: @@ -92,7 +92,7 @@ def test_rows_are_keyed_by_the_results_folder_not_the_recorded_source(self) -> N [("blackjack", 1, 1.0), ("blackjack_before_fixes", 1, 0.0)], ) - def test_an_unknown_cost_makes_the_slot_cost_unknown(self) -> None: + def test_an_unknown_cost_makes_the_model_cost_unknown(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) write_job("blackjack", [_episode(0, 1.0, 0.0001), _episode(1, 1.0, None)], results_dir=root) diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index a1459f0..6d99215 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -28,7 +28,7 @@ from s1a.tool import loop, series from support import COUNTER -RANDOM_RUN = ["--slot", "random", "--rethink", "off", "--episodes", "1", "--showcase"] +RANDOM_RUN = ["--model", "random", "--rethink", "off", "--episodes", "1", "--showcase"] NO_KEYS = {"TYPESAFE_API_KEY": "", "OPENROUTER_API_KEY": "", "MODEL_NAME": "", "OPENAI_API_KEY": "", "LLM_API_KEY": ""} HAVE_RLCARD = importlib.util.find_spec("rlcard") is not None # the one tool agent that runs offline HARNESS_LOG = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \| ") # the console entry routes these to files @@ -104,7 +104,7 @@ async def test_the_listing_and_help_leave_the_stream_clean(self) -> None: async def test_a_blackjack_run_leaves_the_stream_clean_and_writes_under_s1a_home(self) -> None: async with self._session() as session: run = await session.call_tool( - "run_agent", {"name": "blackjack", "flags": ["--slot", "rule", *RANDOM_RUN[2:]]} + "run_agent", {"name": "blackjack", "flags": ["--model", "rule", *RANDOM_RUN[2:]]} ) self.assertEqual(self.strays, []) self.assertFalse(run.isError, run.content[0].text if run.content else run) @@ -136,7 +136,7 @@ async def test_exposes_the_three_tools(self) -> None: async def test_decide_returns_the_validated_choice(self) -> None: transport = ScriptedTransport(choose="inc") - with patch.object(mcp_server, "build_model", lambda slot, **kwargs: JevModel(transport)): + with patch.object(mcp_server, "build_model", lambda model_name, **kwargs: JevModel(transport)): async with create_connected_server_and_client_session(mcp_server.server) as session: result = await session.call_tool( "decide", {"state": {"n": 1}, "options": {"inc": "add one", "noop": "do nothing"}, "rules": "count"} @@ -159,7 +159,7 @@ async def test_list_agents_names_every_front_with_its_flags(self) -> None: self.assertIn(f"uv sync --extra {name}", rows[name]["description"]) flags = {name: {row["flag"]: row for row in rows[name]["flags"]} for name in ("game2048", "flights")} self.assertEqual( - (flags["game2048"]["--slot"]["required"], flags["game2048"]["--slot"]["choices"]), + (flags["game2048"]["--model"]["required"], flags["game2048"]["--model"]["choices"]), (True, ["jev", "llm", "random", "rule", "laya", "cua"]), ) max_steps = str(agents.load("game2048").budget.max_steps) @@ -168,7 +168,7 @@ async def test_list_agents_names_every_front_with_its_flags(self) -> None: (False, max_steps), ) self.assertEqual( - (flags["game2048"]["--slot"]["takes_value"], flags["game2048"]["--headed"]["takes_value"]), (True, False) + (flags["game2048"]["--model"]["takes_value"], flags["game2048"]["--headed"]["takes_value"]), (True, False) ) self.assertIsNone(flags["game2048"]["--headed"]["default"]) # a switch is passed alone: no value to send self.assertEqual( diff --git a/tests/test_rails.py b/tests/test_rails.py index c6f18ab..2fa1fff 100644 --- a/tests/test_rails.py +++ b/tests/test_rails.py @@ -249,12 +249,12 @@ def test_the_shipped_labelled_set_is_well_formed_and_balanced(self) -> None: class TestPlay(IsolatedAsyncioTestCase): - def test_the_slot_flag_defaults_to_jev_and_offers_laya(self) -> None: + def test_the_model_flag_defaults_to_jev_and_offers_laya(self) -> None: args = rails.parser(GUARD).parse_args([]) - self.assertEqual((args.slot, args.labelled_set), ("jev", GUARD.labelled_set)) - self.assertEqual(rails.parser(GUARD).parse_args(["--slot", "laya"]).slot, "laya") + self.assertEqual((args.model, args.labelled_set), ("jev", GUARD.labelled_set)) + self.assertEqual(rails.parser(GUARD).parse_args(["--model", "laya"]).model, "laya") with self.assertRaises(SystemExit): - rails.parser(GUARD).parse_args(["--slot", "random"]) + rails.parser(GUARD).parse_args(["--model", "random"]) async def test_play_warms_evaluates_and_closes_the_slot_model(self) -> None: events: list[str] = [] @@ -268,11 +268,11 @@ async def close(self) -> None: built: list[str] = [] - def build(slot: str, **kwargs: Any) -> ScriptedModel: - built.append(slot) + def build(model_name: str, **kwargs: Any) -> ScriptedModel: + built.append(model_name) return Recording(noul=[0.9] * 20) with tempfile.TemporaryDirectory() as tmp, patch.object(rails, "build_model", build): - args = rails.parser(GUARD).parse_args(["--slot", "laya"]) + args = rails.parser(GUARD).parse_args(["--model", "laya"]) summary = await rails.play(GUARD, args, results_dir=Path(tmp)) self.assertEqual((built, events, summary["records"]), (["laya"], ["warm", "close"], 20)) diff --git a/tests/test_replay.py b/tests/test_replay.py index aef758a..f545dc2 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -116,10 +116,10 @@ def _write_pair(root: Path, *, with_views: bool = True) -> tuple[Path, Path]: def _write_browser_pair(root: Path, *, png: bytes) -> tuple[Path, Path]: - """A browser run per slot as ``s1a run --slot ...`` and ``evals.replay.cast`` leave them: answer.json, the - slot's records and two stamped frames each.""" + """A browser run per model as ``s1a run --model ...`` and ``evals.replay.cast`` leave them: answer.json, the + model's records and two stamped frames each.""" jev, llm = root / "jev-1", root / "llm-1" - for folder, slot, elapsed_ms, cost in ((jev, "jev", 3000, 0.01), (llm, "llm", 4000, 0.5)): + for folder, model_name, elapsed_ms, cost in ((jev, "jev", 3000, 0.01), (llm, "llm", 4000, 0.5)): (folder / "frames").mkdir(parents=True) answer = { "ok": True, @@ -132,7 +132,7 @@ def _write_browser_pair(root: Path, *, png: bytes) -> tuple[Path, Path]: "title": "Easy Vegetarian Spinach Lasagna", }, "agent": "allrecipes", - "slot": slot, + "model": model_name, } (folder / "answer.json").write_text(json.dumps(answer), encoding="utf-8") for name in ("t00000500-0001-browser_navigate.png", "t00001500-0002-browser_run_code_unsafe.png"): @@ -191,7 +191,7 @@ def test_a_decision_model_run_reads_its_ticks_on_the_policy_clock(self) -> None: with tempfile.TemporaryDirectory() as tmp: jev_dir, _ = _write_browser_pair(Path(tmp), png=b"png") trial = read_trial(jev_dir) - self.assertEqual((trial.eval_name, trial.slot, trial.score, trial.elapsed_s), ("allrecipes", "jev", 1.0, 3.0)) + self.assertEqual((trial.eval_name, trial.model, trial.score, trial.elapsed_s), ("allrecipes", "jev", 1.0, 3.0)) self.assertEqual((trial.steps, trial.cost_usd, trial.rejected), (2, 0.01, 0)) self.assertEqual([d["key"] for d in trial.decisions], ["TYPE_TEXT · Search the site", "DONE"]) self.assertEqual([d["ms"] for d in trial.decisions], [400, 300]) @@ -202,11 +202,21 @@ def test_a_decision_model_run_reads_its_ticks_on_the_policy_clock(self) -> None: self.assertEqual([f.name[:10] for f in trial.frames], ["t00000500-", "t00001500-"]) self.assertEqual((trial.extra["front"], len(trial.extra["history"])), ("browser", 1)) + def test_a_run_written_by_0_1_0_names_the_model_under_slot(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + jev_dir, _ = _write_browser_pair(Path(tmp), png=b"png") + answer_file = jev_dir / "answer.json" + answer = json.loads(answer_file.read_text(encoding="utf-8")) + answer["slot"] = answer.pop("model") + answer_file.write_text(json.dumps(answer), encoding="utf-8") + trial = read_trial(jev_dir) + self.assertEqual((trial.model, trial.steps), ("jev", 2)) + def test_a_chat_model_run_counts_the_calls_that_issued_tool_calls(self) -> None: with tempfile.TemporaryDirectory() as tmp: _, llm_dir = _write_browser_pair(Path(tmp), png=b"png") trial = read_trial(llm_dir) - self.assertEqual((trial.slot, trial.steps, trial.elapsed_s, trial.cost_usd), ("llm", 1, 4.0, 0.5)) + self.assertEqual((trial.model, trial.steps, trial.elapsed_s, trial.cost_usd), ("llm", 1, 4.0, 0.5)) self.assertEqual( [d["key"] for d in trial.decisions], ["navigate · https://www.allrecipes.com/search?q=lasagna"] ) @@ -220,7 +230,7 @@ def test_the_pair_page_draws_the_frame_at_the_clock(self) -> None: html = render_page([read_trial(llm_dir), read_trial(jev_dir)], out_dir=out) copied = sorted(p.name for p in (out / "frames-jev").iterdir()) data = _page_data(html) - self.assertEqual([t["slot"] for t in data["trials"]], ["jev", "llm"]) + self.assertEqual([t["model"] for t in data["trials"]], ["jev", "llm"]) self.assertEqual([t["front"] for t in data["trials"]], ["browser", "browser"]) self.assertEqual(data["trials"][0]["frames"], [f"frames-jev/{name}" for name in copied]) self.assertEqual(data["trials"][0]["times"], [0, 1000, 3000]) @@ -233,7 +243,7 @@ def test_read_trial_reads_the_harbor_folder(self) -> None: jev_dir, _ = _write_pair(Path(tmp)) trial = read_trial(jev_dir) self.assertIsInstance(trial, Trial) - self.assertEqual((trial.eval_name, trial.slot, trial.seed, trial.score), ("blackjack", "jev", 0, 1.0)) + self.assertEqual((trial.eval_name, trial.model, trial.seed, trial.score), ("blackjack", "jev", 0, 1.0)) self.assertEqual((trial.steps, trial.elapsed_s, trial.cost_usd), (2, 1.0, 0.00004)) self.assertEqual([d["key"] for d in trial.decisions], ["hit", "stand"]) self.assertEqual(len(trial.views), 3) @@ -324,7 +334,7 @@ def test_blackjack_views_are_rebuilt_from_the_final_hand_when_missing(self) -> N class TestPage(TestCase): - def test_pair_page_holds_both_slots_and_every_step(self) -> None: + def test_pair_page_holds_both_models_and_every_step(self) -> None: with tempfile.TemporaryDirectory() as tmp: jev_dir, llm_dir = _write_pair(Path(tmp)) html = render_page([read_trial(jev_dir), read_trial(llm_dir)], out_dir=Path(tmp) / "out") @@ -332,13 +342,13 @@ def test_pair_page_holds_both_slots_and_every_step(self) -> None: html.split('", 1)[0] ) self.assertEqual(data["eval"], "blackjack") - self.assertEqual([t["slot"] for t in data["trials"]], ["jev", "llm"]) + self.assertEqual([t["model"] for t in data["trials"]], ["jev", "llm"]) self.assertEqual([t["times"] for t in data["trials"]], [[0, 500, 1000], [0, 500, 1000]]) self.assertEqual(len(data["trials"][0]["views"]), 3) self.assertEqual(data["trials"][0]["decisions"][0]["probabilities"], {"hit": 0.8, "stand": 0.2}) self.assertIn("replay.setTime", html) self.assertIn("replay.setStep", html) - self.assertIn('"System 1 · " + trial.slot', html) + self.assertIn('"System 1 · " + trial.model', html) def test_jev_is_always_left_and_llm_right(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -347,7 +357,7 @@ def test_jev_is_always_left_and_llm_right(self) -> None: data = json.loads( html.split('", 1)[0] ) - self.assertEqual([t["slot"] for t in data["trials"]], ["jev", "llm"]) + self.assertEqual([t["model"] for t in data["trials"]], ["jev", "llm"]) self.assertIn("blackjack: jev vs llm", html) def test_page_copies_frames_next_to_itself(self) -> None: diff --git a/tests/test_spec.py b/tests/test_spec.py index 0b4b6eb..f4191ed 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -81,10 +81,10 @@ def test_load_rejects_a_spec_whose_name_differs_from_its_module(self) -> None: class TestSharedFlags(TestCase): def test_budget_is_the_default_and_the_agents_flags_come_after(self) -> None: spec = replace(COUNTER, flags=stride_flag) - args = series.parser(spec).parse_args(["--slot", "random", "--rethink", "off", "--episodes", "2"]) + args = series.parser(spec).parse_args(["--model", "random", "--rethink", "off", "--episodes", "2"]) self.assertEqual((args.max_steps, args.timeout, args.stride, args.headed, args.seed), (5, 30.0, 1, False, 0)) args = series.parser(spec).parse_args( - ["--slot", "random", "--rethink", "off", "--episodes", "2", "--max-steps", "9", "--stride", "3"] + ["--model", "random", "--rethink", "off", "--episodes", "2", "--max-steps", "9", "--stride", "3"] ) self.assertEqual((args.max_steps, args.stride), (9, 3)) @@ -102,13 +102,13 @@ def test_episodes_max_steps_and_timeout_at_or_below_zero_are_usage_errors(self) ["--episodes", "1", "--timeout", "-2.5"], ): with self.assertRaises(SystemExit) as caught: - series.parser(COUNTER).parse_args(["--slot", "random", "--rethink", "off", *flags]) + series.parser(COUNTER).parse_args(["--model", "random", "--rethink", "off", *flags]) self.assertEqual(caught.exception.code, 2) class TestTemplatePlaysThroughTheRunner(IsolatedAsyncioTestCase): - async def test_random_slot_plays_two_episodes_and_writes_one_job_folder(self) -> None: - args = series.parser(COUNTER).parse_args(["--slot", "random", "--rethink", "off", "--episodes", "2"]) + async def test_random_plays_two_episodes_and_writes_one_job_folder(self) -> None: + args = series.parser(COUNTER).parse_args(["--model", "random", "--rethink", "off", "--episodes", "2"]) with ( tempfile.TemporaryDirectory() as tmp, patch.object(loop, "WORKSPACE", Path(tmp) / "ws"), @@ -125,7 +125,7 @@ async def test_random_slot_plays_two_episodes_and_writes_one_job_folder(self) -> async def test_a_series_that_selects_no_seeds_is_a_run_error(self) -> None: spec = replace(COUNTER, series=lambda flags: replace(counter_series(flags), seeds=())) - args = series.parser(spec).parse_args(["--slot", "random", "--rethink", "off", "--episodes", "1"]) + args = series.parser(spec).parse_args(["--model", "random", "--rethink", "off", "--episodes", "1"]) with ( tempfile.TemporaryDirectory() as tmp, patch.object(series, "optional_chat_model", lambda: None), @@ -135,7 +135,7 @@ async def test_a_series_that_selects_no_seeds_is_a_run_error(self) -> None: await series.play(spec, args, results_dir=Path(tmp) / "results") async def test_a_series_without_chat_tokens_never_looks_a_price_up(self) -> None: - args = series.parser(COUNTER).parse_args(["--slot", "rule", "--rethink", "off", "--episodes", "1"]) + args = series.parser(COUNTER).parse_args(["--model", "rule", "--rethink", "off", "--episodes", "1"]) with ( tempfile.TemporaryDirectory() as tmp, patch.object(loop, "WORKSPACE", Path(tmp) / "ws"), @@ -156,7 +156,7 @@ def slow_series(flags: Any) -> Series: def slow_pricing(episodes: list[Any]) -> None: on_main["pricing"] = threading.current_thread() is threading.main_thread() - args = series.parser(COUNTER).parse_args(["--slot", "rule", "--rethink", "off", "--episodes", "1"]) + args = series.parser(COUNTER).parse_args(["--model", "rule", "--rethink", "off", "--episodes", "1"]) with ( tempfile.TemporaryDirectory() as tmp, patch.object(loop, "WORKSPACE", Path(tmp) / "ws"), @@ -179,7 +179,7 @@ def flaky_series(flags: Any) -> Series: annotate=lambda env, episode: None, ) - args = series.parser(COUNTER).parse_args(["--slot", "rule", "--rethink", "off", "--episodes", "2"]) + args = series.parser(COUNTER).parse_args(["--model", "rule", "--rethink", "off", "--episodes", "2"]) with ( tempfile.TemporaryDirectory() as tmp, patch.object(loop, "WORKSPACE", Path(tmp) / "ws"), @@ -198,8 +198,8 @@ def never(flags: Any) -> Series: raise AssertionError("the series must not be built before the run's keys are checked") spec = replace(COUNTER, series=never) - for slot, env in (("jev", {}), ("llm", {}), ("rule", {"CHAT_USD_PER_M_INPUT": "0.3"})): - args = series.parser(COUNTER).parse_args(["--slot", slot, "--rethink", "off", "--episodes", "1"]) + for model_name, env in (("jev", {}), ("llm", {}), ("rule", {"CHAT_USD_PER_M_INPUT": "0.3"})): + args = series.parser(COUNTER).parse_args(["--model", model_name, "--rethink", "off", "--episodes", "1"]) with ( patch.dict(os.environ, env, clear=True), patch.object(series, "optional_chat_model", lambda: None), @@ -208,7 +208,7 @@ def never(flags: Any) -> Series: await series.play(spec, args, results_dir=Path("unused")) async def test_a_series_whose_every_episode_fails_writes_its_job_and_raises(self) -> None: - args = series.parser(COUNTER).parse_args(["--slot", "jev", "--rethink", "off", "--episodes", "2"]) + args = series.parser(COUNTER).parse_args(["--model", "jev", "--rethink", "off", "--episodes", "2"]) with ( tempfile.TemporaryDirectory() as tmp, patch.object(loop, "WORKSPACE", Path(tmp) / "ws"), @@ -228,7 +228,7 @@ def _no_lookup(model_name: str) -> None: raise AssertionError("a series that spent no chat tokens must not fetch the price catalogue") -def _refusing(slot: str, **kwargs: Any) -> JevModel: +def _refusing(model_name: str, **kwargs: Any) -> JevModel: """The jev model of a run whose key is wrong or whose endpoint is down.""" error = build_error(StatusCode.MODEL_CALL_FAILED, error_msg="decisions endpoint returned HTTP 401") return JevModel(ScriptedTransport(error=error)) diff --git a/tests/test_templates.py b/tests/test_templates.py index 14b4e39..4353f8c 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -22,8 +22,8 @@ from s1a.tool import loop, series from support import POLICY, browse_offline -RANDOM = ["--slot", "random", "--rethink", "off", "--episodes", "3"] -RULE = ["--slot", "rule", "--rethink", "off", "--episodes", "3"] +RANDOM = ["--model", "random", "--rethink", "off", "--episodes", "3"] +RULE = ["--model", "rule", "--rethink", "off", "--episodes", "3"] class TestNimEnv(IsolatedAsyncioTestCase): @@ -64,7 +64,7 @@ async def _play(self, flags: list[str]) -> dict: result = await series.play(tool_agent.SPEC, args, results_dir=Path(tmp) / "results") return json.loads((Path(result["job_dir"]) / "summary.json").read_text(encoding="utf-8")) - async def test_the_rule_slot_wins_every_game_and_the_random_slot_plays_to_the_end(self) -> None: + async def test_rule_wins_every_game_and_random_plays_to_the_end(self) -> None: rule = await self._play(RULE) self.assertEqual( (rule["policy"], rule["episodes"], rule["mean_score"], rule["invalid_keys"]), ("winning", 3, 1.0, 0) @@ -79,7 +79,7 @@ class TestBrowserTemplateOffline(IsolatedAsyncioTestCase): async def test_the_spec_reaches_the_subagent(self) -> None: spec = browser_agent.SPEC - answer, seen, files = await browse_offline(spec, POLICY, slot="jev", max_steps=spec.budget.max_steps) + answer, seen, files = await browse_offline(spec, POLICY, model_name="jev", max_steps=spec.budget.max_steps) self.assertIsInstance(seen["model"], BrowserDecisionModel) self.assertEqual((seen["max_iterations"], seen["language"]), (spec.budget.max_steps, spec.language)) self.assertEqual((answer["ok"], answer["final"], files), (True, "Three flights.", ["decision_ticks.json"])) diff --git a/tests/test_tool_loop.py b/tests/test_tool_loop.py index caea64b..ab63868 100644 --- a/tests/test_tool_loop.py +++ b/tests/test_tool_loop.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""The eval agent's two tools, the slot factory, and one episode through the DeepAgent offline.""" +"""The eval agent's two tools, the slot-model factory, and one episode through the DeepAgent offline.""" from __future__ import annotations @@ -78,7 +78,7 @@ async def step(self, key: str) -> None: class ScriptedChatModel(Model): - """The llm slot offline: one act call per decision turn from a script of keys, then a final line.""" + """The chat model offline: one act call per decision turn from a script of keys, then a final line.""" def __init__(self, keys: list[str]) -> None: source = placeholder_model() @@ -197,7 +197,7 @@ def test_llm_ticks_split_one_reply_with_two_act_calls_and_survive_an_uninvoked_c class TestSlotFactory(TestCase): - def test_every_model_slot_is_the_one_slot_model_and_the_errors(self) -> None: + def test_every_name_but_llm_is_the_one_slot_model_and_the_errors(self) -> None: env, state = CountingEnv(), EvalState() chance = build_slot_model("random", env, state, rules=RULES, chat=None, decision_model=RandomModel(0)) self.assertIsInstance(chance, ToolDecisionModel) @@ -205,7 +205,7 @@ def test_every_model_slot_is_the_one_slot_model_and_the_errors(self) -> None: rule = build_slot_model("rule", env, state, rules=RULES, chat=None, decision_model=ALWAYS_INC) self.assertIsInstance(rule, ToolDecisionModel) self.assertEqual(rule.name, "always-inc") - for slot, error in ( + for model_name, error in ( ("llm", RuntimeError), ("rule", RuntimeError), ("jev", RuntimeError), @@ -213,7 +213,7 @@ def test_every_model_slot_is_the_one_slot_model_and_the_errors(self) -> None: ("oracle", ValueError), ): with self.assertRaises(error): - build_slot_model(slot, env, state, rules=RULES, chat=None, decision_model=None) + build_slot_model(model_name, env, state, rules=RULES, chat=None, decision_model=None) async def _play( @@ -221,7 +221,7 @@ async def _play( *, max_acts: int, timeout_s: float, - slot: str, + model_name: str, decision_model: DecisionModel | None, chat: Model | None = None, rethink_on: bool = False, @@ -232,7 +232,7 @@ async def _play( return await run_episode( SPEC, env, - slot=slot, + model_name=model_name, seed=0, chat=chat, decision_model=decision_model, @@ -255,14 +255,14 @@ class TestEpisodeThroughTheAgent(IsolatedAsyncioTestCase): """Episodes through ``create_deep_agent`` and the Runner, offline: a rule in the slot, no chat model.""" async def test_a_refused_decision_is_the_episodes_error_with_no_decisions(self) -> None: - episode = await _play(CountingEnv(), max_acts=10, timeout_s=60.0, slot="jev", decision_model=_refusing()) + episode = await _play(CountingEnv(), max_acts=10, timeout_s=60.0, model_name="jev", decision_model=_refusing()) self.assertTrue(episode.error.startswith("decision failed: "), episode.error) self.assertIn("decisions endpoint returned HTTP 401", episode.error) self.assertEqual((episode.decisions, episode.steps, episode.score), ([], 0, 0.0)) self.assertIn("BLOCKED", episode.extra["output"]) - async def test_rule_slot_plays_to_the_end_and_every_act_is_a_recorded_decision(self) -> None: - episode = await _play(CountingEnv(), max_acts=10, timeout_s=60.0, slot="rule", decision_model=ALWAYS_INC) + async def test_rule_plays_to_the_end_and_every_act_is_a_recorded_decision(self) -> None: + episode = await _play(CountingEnv(), max_acts=10, timeout_s=60.0, model_name="rule", decision_model=ALWAYS_INC) self.assertEqual((episode.policy, episode.score, episode.steps), ("always-inc", 3.0, 3)) self.assertEqual([decision["key"] for decision in episode.decisions], ["inc", "inc", "inc"]) self.assertEqual(episode.final_state, {"n": 3}) @@ -278,26 +278,36 @@ async def test_rule_slot_plays_to_the_end_and_every_act_is_a_recorded_decision(s self.assertIsNone(episode.frames_dir) async def test_two_random_runs_with_the_same_seed_pick_the_same_keys(self) -> None: - first = await _play(CountingEnv(), max_acts=10, timeout_s=60.0, slot="random", decision_model=RandomModel(7)) - second = await _play(CountingEnv(), max_acts=10, timeout_s=60.0, slot="random", decision_model=RandomModel(7)) + first = await _play( + CountingEnv(), max_acts=10, timeout_s=60.0, model_name="random", decision_model=RandomModel(7) + ) + second = await _play( + CountingEnv(), max_acts=10, timeout_s=60.0, model_name="random", decision_model=RandomModel(7) + ) self.assertEqual([d["key"] for d in first.decisions], [d["key"] for d in second.decisions]) self.assertEqual((first.policy, first.decisions[0]["probabilities"]), ("random", {"inc": 0.5, "noop": 0.5})) async def test_the_act_budget_stops_the_agent_before_the_game_ends(self) -> None: - episode = await _play(CountingEnv(), max_acts=2, timeout_s=60.0, slot="rule", decision_model=ALWAYS_INC) + episode = await _play(CountingEnv(), max_acts=2, timeout_s=60.0, model_name="rule", decision_model=ALWAYS_INC) self.assertEqual((episode.score, episode.steps), (2.0, 2)) self.assertEqual(episode.extra["result_type"], "answer") async def test_an_env_failure_inside_act_ends_the_episode_as_an_error(self) -> None: - episode = await _play(DyingEnv(), max_acts=5, timeout_s=60.0, slot="rule", decision_model=ALWAYS_INC) + episode = await _play(DyingEnv(), max_acts=5, timeout_s=60.0, model_name="rule", decision_model=ALWAYS_INC) self.assertEqual(episode.error, "act failed: playwright died") self.assertEqual((episode.steps, len(episode.decisions), episode.score), (0, 1, 0.0)) self.assertIn("BLOCKED", episode.extra["output"]) - async def test_the_llm_slot_marks_rejected_keys_and_runs_without_the_rail(self) -> None: + async def test_llm_marks_rejected_keys_and_runs_without_the_rail(self) -> None: chat = ScriptedChatModel(["dec", "inc", "inc", "inc"]) episode = await _play( - CountingEnv(), max_acts=10, timeout_s=60.0, slot="llm", decision_model=None, chat=chat, rethink_on=True + CountingEnv(), + max_acts=10, + timeout_s=60.0, + model_name="llm", + decision_model=None, + chat=chat, + rethink_on=True, ) self.assertEqual((episode.policy, episode.score, episode.steps, episode.invalid_keys), ("llm", 3.0, 3, 1)) self.assertEqual( @@ -315,7 +325,7 @@ async def test_an_episode_leaves_no_system_tools_or_conversation_behind(self) -> await run_episode( SPEC, CountingEnv(), - slot="rule", + model_name="rule", seed=0, chat=None, decision_model=ALWAYS_INC, @@ -334,7 +344,7 @@ async def test_an_episode_leaves_no_system_tools_or_conversation_behind(self) -> self.assertEqual([store for store in stores if store.startswith("evals-counter")], []) async def test_a_slow_episode_stops_at_the_timeout_and_keeps_its_score(self) -> None: - episode = await _play(SlowEnv(), max_acts=10, timeout_s=0.5, slot="rule", decision_model=ALWAYS_INC) + episode = await _play(SlowEnv(), max_acts=10, timeout_s=0.5, model_name="rule", decision_model=ALWAYS_INC) self.assertEqual(episode.extra["result_type"], "timeout") self.assertLess(episode.score, 3.0) self.assertIn(len(episode.decisions) - episode.steps, (0, 1), "the decision in flight at the cut has no act") diff --git a/tests/test_tool_models.py b/tests/test_tool_models.py index f3920ec..3ebe221 100644 --- a/tests/test_tool_models.py +++ b/tests/test_tool_models.py @@ -188,7 +188,7 @@ async def test_stream_yields_the_act_call_as_one_chunk(self) -> None: self.assertEqual(chunks[0].finish_reason, "tool_calls") -class TestOtherSlots(IsolatedAsyncioTestCase): +class TestOtherModels(IsolatedAsyncioTestCase): async def test_random_picks_an_offered_key_with_a_uniform_distribution(self) -> None: state = EvalState() message = await _model(CountingEnv(), state, RandomModel(1)).invoke([], tools=TOOLS)