From d46edab3f778b0dde3b7925494069f291e090350 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Wed, 11 Nov 2026 21:21:47 -0500 Subject: [PATCH] feat(store): show the active store in doctor, status and a SessionStart draft `commons doctor` gains a `store` check and `commons status` opens with a `store:` line: `local database on `, `hosted at `, or none without a config. Neither shows a key or password, and both go through `redact`. Doctor's server line is scrubbed the same way. Unshipped: the `store-line` variant appends "Store for this session: {store}." to the shipped SessionStart line (the hook fills in the session's bound store), and `lesson-store` pairs it with `lesson-also-propose` as the eval condition. The eval's per-session database names are now opaque hashes, since the store line shows the agent that name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- docs/eval/README.md | 31 ++++ server/src/commons/cli/hooks.py | 13 +- server/src/commons/cli/main.py | 8 + server/src/commons/service/agent_text.py | 12 ++ server/src/commons/service/doctor.py | 25 ++- server/src/commons/service/identity.py | 25 ++- server/src/commons/store/connection.py | 9 ++ server/tests/test_active_store.py | 185 +++++++++++++++++++++++ server/tests/test_status.py | 3 +- server/tests/test_variants.py | 41 +++++ tools/eval/runner.py | 10 +- 11 files changed, 351 insertions(+), 11 deletions(-) create mode 100644 server/tests/test_active_store.py diff --git a/docs/eval/README.md b/docs/eval/README.md index 5899f29..d304de6 100644 --- a/docs/eval/README.md +++ b/docs/eval/README.md @@ -825,6 +825,37 @@ two known-entry scenarios (and no duplicate proposal there), searches and propos controls, rejected `propose` calls, and fix rate. The bar is decision 0023's. MIT-598 holds the go/no-go. +## Drafted variant: the session's store (MIT-653, 2026-11-11) + +In two real misses the agent used the wrong store, or wrote the lesson to AGENTS.md instead of +the commons. Nothing ships: `baseline` and the hook texts are unchanged, and the store line's +hook version joins `RELEASED_HOOK_TEXTS` only if it ships. + +| Variant | Surface | Change | +|---|---|---| +| `store-line` | SessionStart line | The shipped line plus "Store for this session: {store}.", the hook filling in `hosted at ` or `local database on ` (`identity.store_label`: no key, no password) | +| `lesson-store` | instructions and SessionStart line | `lesson-also-propose` and `store-line` together: the condition to run | + +Eval sessions run on a fresh local database each, and the store line shows the agent its name, +so `runner.database_name` is now an opaque hash (`commons_<16 hex>`): the old name spelled out +the condition and scenario. A `--store NAME` experiment store is shown under the name you give it. + +Not run yet (the owner starts each batch). On the four no-test scenarios and the five genuine +controls, 3 runs each, on both models: 9 scenarios x 3 runs x 2 conditions x 2 models = +**108 sessions**. + +``` +scenarios=(--scenario config-app-overwrites --scenario config-default-wins + --scenario known-config-app-overwrites --scenario known-config-default-wins + --scenario control-typo --scenario control-arithmetic --scenario control-clear-error + --scenario control-documented-option --scenario control-documented-flag --runs 3) +for model in sonnet claude-opus-5-5; do + uv run python tools/agent_eval.py run "${scenarios[@]}" --model $model --condition st-baseline-$model + uv run python tools/agent_eval.py run "${scenarios[@]}" --model $model --variant lesson-store --condition st-lesson-store-$model +done +uv run python tools/agent_eval.py report --condition st-baseline-sonnet --condition st-lesson-store-sonnet +``` + ## Hold-out change: whenever retired, Polars added (2026-11-08) **What changed.** `real-whenever-ignore-dst` is out of the measured hold-out set, and diff --git a/server/src/commons/cli/hooks.py b/server/src/commons/cli/hooks.py index 91f759a..cc908a3 100644 --- a/server/src/commons/cli/hooks.py +++ b/server/src/commons/cli/hooks.py @@ -14,7 +14,7 @@ from commons.service import agent_text, hook_client from commons.service.hook_client import HookBackend, LocalBackend, RemoteBackend, remote_client from commons.service.hooks import StopResult -from commons.service.identity import hosted_server +from commons.service.identity import hosted_server, store_label from commons.service.local_state import bind_store, bound_store, state_dir from commons.store.connection import connect @@ -43,7 +43,7 @@ def _session_start(payload: Payload) -> HookOutput: context = agent_text.SESSION_START.format(task_id=task_id) start_line = shown.start_line if start_line: - context = f"{context} {start_line}" + context = f"{context} {start_line.format(store=_session_store(session_id))}" if feedback := backend.feedback(): # off unless the store turns it on (decision 0035) context = f"{context} {feedback}" return {"hookSpecificOutput": {"hookEventName": "SessionStart", "additionalContext": context}} @@ -63,6 +63,15 @@ def _observe(use: ToolUse) -> None: hook_client.on_tool_use(backend_for(use.session_id), use) +def _session_store(session_id: str) -> str: + """Name the store this session's hooks use, as `backend_for` picks it, without secrets.""" + bound = bound_store(session_id) + if bound is None: + current = _current_server() + return store_label(None if current is None else current[0]) + return store_label(None if bound["store"] == "local" else bound["store"]) + + def _current_server() -> tuple[str, str] | None: try: return hosted_server() diff --git a/server/src/commons/cli/main.py b/server/src/commons/cli/main.py index e1d0b95..2925b8f 100644 --- a/server/src/commons/cli/main.py +++ b/server/src/commons/cli/main.py @@ -53,9 +53,11 @@ from commons.service.hook_install import default_settings_path, install_hooks from commons.service.identity import ( acting_as, + active_store, hosted_server_or_none, init_reporter, save_server, + store_label, ) from commons.store import insights from commons.store.connection import connect @@ -131,6 +133,12 @@ def _pending(_args: argparse.Namespace) -> int: def _status(_args: argparse.Namespace) -> int: + # The store goes first, before the database is reached, so it shows even when that fails. + store, local = active_store(), store_label(None) + unset = "none configured; run `commons init` (local) or `commons join` (hosted)" + sys.stdout.write(f"store: {store or unset}\n") + if store != local: # the statistics are always the local database's + sys.stdout.write(f"statistics from: {local}\n") with connect() as conn: result = status_service.status(conn) counts = ", ".join(f"{count} {status}" for status, count in result.entries_by_status.items()) diff --git a/server/src/commons/service/agent_text.py b/server/src/commons/service/agent_text.py index daf25e0..7cacaf0 100644 --- a/server/src/commons/service/agent_text.py +++ b/server/src/commons/service/agent_text.py @@ -576,6 +576,18 @@ def _draft(edits: list[tuple[str, str]], propose: str) -> Variant: VARIANTS["propose-example"] = _draft([], PROPOSE_EXAMPLES_APP) VARIANTS["combined"] = _draft(list(_DRAFT_EDITS.values()), PROPOSE_EXAMPLES_APP) +# Store visibility (MIT-653), not shipped: in two real misses the agent used the wrong store, +# or wrote the lesson to AGENTS.md instead of the commons. The SessionStart context names the +# session's store after the shipped start line; the hook fills `{store}` with +# `identity.store_label` (a hosted server's host, or the local database's name, never a secret). +# Shipping it changes what installed hooks show, so its hook version joins RELEASED_HOOK_TEXTS +# only then. +STORE_LINE = "Store for this session: {store}." +_STORE_START_LINE = f"{VARIANTS['baseline'].start_line} {STORE_LINE}" +VARIANTS["store-line"] = replace(VARIANTS["baseline"], start_line=_STORE_START_LINE) +# The eval condition that pairs it with `lesson-also-propose`: both aim at the AGENTS.md miss. +VARIANTS["lesson-store"] = replace(VARIANTS["lesson-also-propose"], start_line=_STORE_START_LINE) + def _variant_name() -> str: return os.environ.get("COMMONS_INSTRUCTIONS_VARIANT", "baseline") diff --git a/server/src/commons/service/doctor.py b/server/src/commons/service/doctor.py index f64f475..5014110 100644 --- a/server/src/commons/service/doctor.py +++ b/server/src/commons/service/doctor.py @@ -17,8 +17,15 @@ NotPermittedError, StoreUnavailableError, ) +from commons.privacy.redaction import scrub from commons.service.hook_install import default_settings_path, is_commons_hook -from commons.service.identity import config_path, current_reporter, hosted_server, read_config +from commons.service.identity import ( + active_store, + config_path, + current_reporter, + hosted_server, + read_config, +) from commons.service.local_state import active_sessions, state_dir from commons.store.connection import Connection from commons.store.migrate import pending_migrations @@ -42,7 +49,7 @@ class Check: def run_checks(connect: Connect, ping: Ping) -> list[Check]: """Run every check; a failure in one doesn't stop the others.""" - checks = [_config()] + checks = [_config(), _store()] server = _hosted() if server is None: checks += _local_database(connect) @@ -63,6 +70,16 @@ def _config() -> Check: return Check("config", "ok", f"{path} ({mode} mode)") +def _store() -> Check: + """Name the store searches, proposals and hooks go to, so a wrong one is plain to see.""" + store = active_store() + if store is None: + return Check( + "store", "warn", "none until `commons init` (local) or `commons join` (hosted)" + ) + return Check("store", "ok", store) + + def _hosted() -> tuple[str, str] | None: try: return hosted_server() @@ -96,8 +113,8 @@ def _server(server: tuple[str, str], ping: Ping) -> Check: try: ping(url, key) except (NotInitializedError, NotPermittedError, httpx2.HTTPError) as error: - return Check("server", "fail", f"{url}: {error}") - return Check("server", "ok", f"{url} accepts your key") + return Check("server", "fail", scrub(f"{url}: {error}")) # a URL may carry a password + return Check("server", "ok", scrub(f"{url} accepts your key")) def _hooks() -> Check: diff --git a/server/src/commons/service/identity.py b/server/src/commons/service/identity.py index ccfaa6f..d4ac3dc 100644 --- a/server/src/commons/service/identity.py +++ b/server/src/commons/service/identity.py @@ -11,11 +11,13 @@ from contextvars import ContextVar from dataclasses import dataclass from pathlib import Path +from urllib.parse import urlsplit from uuid import UUID from commons.domain.errors import NotInitializedError, NotPermittedError from commons.domain.reporters import PrivacyDefault, validate_pseudonym -from commons.store.connection import Connection +from commons.privacy.redaction import scrub +from commons.store.connection import Connection, database_label from commons.store.models import model_id from commons.store.reporters import find_reporter, pseudonym_of, upsert_reporter @@ -191,3 +193,24 @@ def hosted_server_or_none() -> tuple[str, str] | None: return hosted_server() except NotInitializedError: return None + + +def store_label(server: str | None) -> str: + """Name a store without its secrets: `hosted at ` or `local database on `. + + `server` is a hosted server's URL, or None for the local database. Neither the API key nor + a password in either URL is shown, and the result goes through `redact` as well. + """ + if server is None: + return scrub(f"local database {database_label()}") + address = urlsplit(server).netloc.rpartition("@")[2] or server + return scrub(f"hosted at {address}") + + +def active_store() -> str | None: + """Name the store this machine's commands and new sessions use; None without a config.""" + try: + server = hosted_server() + except NotInitializedError: + return None + return store_label(None if server is None else server[0]) diff --git a/server/src/commons/store/connection.py b/server/src/commons/store/connection.py index f4c2213..a347f4c 100644 --- a/server/src/commons/store/connection.py +++ b/server/src/commons/store/connection.py @@ -17,6 +17,15 @@ def database_url() -> str: return os.environ.get("COMMONS_DATABASE_URL", DEFAULT_DATABASE_URL) +def database_label() -> str: + """Name the configured database as ` on `, never with its user or password.""" + try: + params = conninfo_to_dict(database_url()) + except psycopg.ProgrammingError: + return "from an unreadable COMMONS_DATABASE_URL" + return f"{params.get('dbname', '?')} on {params.get('host', '?')}" + + def connect() -> Connection: """Open a connection to the configured database, or raise StoreUnavailableError.""" try: diff --git a/server/tests/test_active_store.py b/server/tests/test_active_store.py new file mode 100644 index 0000000..1a58e1b --- /dev/null +++ b/server/tests/test_active_store.py @@ -0,0 +1,185 @@ +"""Which store is active, shown to the owner (`doctor`, `status`) and in a SessionStart variant. + +Two real misses came from not knowing it: an agent used the wrong store, or wrote the lesson to +AGENTS.md instead of the commons. Every place names the store without a key or password. +""" + +import io +import json +import os +import uuid +from collections.abc import Callable +from pathlib import Path + +import pytest +from psycopg.conninfo import conninfo_to_dict + +from commons.cli.main import main +from commons.domain.errors import StoreUnavailableError +from commons.service import agent_text, doctor +from commons.service.identity import save_server, store_label +from commons.store.connection import Connection + +KEY = "sk-ant-" + "k" * 30 # what a hosted key looks like to `redact` +HIDDEN = "hunter2-not-shown" # the password in the server URL +SERVER = f"https://alice:{HIDDEN}@commons.example.com:8443/" +TASK_ID = uuid.UUID("6f1d2c3b-4a59-4e68-8d7c-0b1a2f3e4d5c") + + +Setup = Callable[[str], None] | None + + +@pytest.fixture +def machine(database_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str: + """An isolated machine on the test database; return that database's name.""" + monkeypatch.setenv("COMMONS_DATABASE_URL", database_url) + monkeypatch.setenv("COMMONS_CONFIG", str(tmp_path / "config.json")) + monkeypatch.setenv("COMMONS_STATE_DIR", str(tmp_path / "state")) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "claude")) + return str(conninfo_to_dict(database_url)["dbname"]) + + +def local(machine: str) -> None: + del machine + assert main(["init", "--pseudonym", f"store-{uuid.uuid4().hex[:8]}"]) == 0 + + +def hosted(machine: str) -> None: + local(machine) + save_server(SERVER, KEY) + + +def no_secrets(text: str) -> None: + assert KEY not in text + assert HIDDEN not in text + assert "alice" not in text + assert os.environ["COMMONS_DATABASE_URL"] not in text + + +def unreachable() -> Connection: + msg = "Can't reach the database at test." + raise StoreUnavailableError(msg) + + +def no_server(_url: str, _key: str) -> None: + return None + + +def test_a_local_store_is_named_by_its_database_and_a_hosted_one_by_its_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + database = f"postgresql://alice:{HIDDEN}@db.internal:5432/commons_team" + monkeypatch.setenv("COMMONS_DATABASE_URL", database) + assert store_label(None) == "local database commons_team on db.internal" + assert store_label(SERVER) == "hosted at commons.example.com:8443" + assert store_label("https://commons.example.com") == "hosted at commons.example.com" + + +def test_a_secret_where_the_host_should_be_is_redacted(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("COMMONS_DATABASE_URL", "not a connection string") + assert store_label(None) == "local database from an unreadable COMMONS_DATABASE_URL" + assert store_label(KEY) == "hosted at [REDACTED:anthropic_key]" # no scheme, so no host + + +@pytest.mark.parametrize( + ("setup", "status", "detail"), + [ + (local, "ok", "local database {db} on localhost"), + (hosted, "ok", "hosted at commons.example.com:8443"), + (None, "warn", "none until `commons init` (local) or `commons join` (hosted)"), + ], +) +def test_doctor_names_the_active_store( + machine: str, + setup: Setup, + status: str, + detail: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + if setup is not None: + setup(machine) + monkeypatch.setattr("commons.service.hook_client.RemoteBackend.ping", lambda _self: None) + (check,) = [c for c in doctor.run_checks(unreachable, no_server) if c.name == "store"] + assert (check.status, check.detail) == (status, detail.format(db=machine)) + main(["doctor"]) + printed = capsys.readouterr().out + assert f" store {detail.format(db=machine)}\n" in printed + no_secrets(printed) + + +@pytest.mark.parametrize( + ("setup", "first_lines"), + [ + (local, ["store: local database {db} on localhost"]), + ( + hosted, + [ + "store: hosted at commons.example.com:8443", + "statistics from: local database {db} on localhost", + ], + ), + ( + None, + [ + "store: none configured; run `commons init` (local) or `commons join` (hosted)", + "statistics from: local database {db} on localhost", + ], + ), + ], +) +def test_status_names_the_active_store_before_its_statistics( + machine: str, setup: Setup, first_lines: list[str], capsys: pytest.CaptureFixture[str] +) -> None: + if setup is not None: + setup(machine) + capsys.readouterr() + assert main(["status"]) == 0 + printed = capsys.readouterr().out + lines = printed.splitlines() + assert lines[: len(first_lines)] == [line.format(db=machine) for line in first_lines] + assert lines[len(first_lines)].startswith("entries: ") + no_secrets(printed) + + +def session_context(monkeypatch: pytest.MonkeyPatch, variant: str) -> str: + monkeypatch.setenv("COMMONS_INSTRUCTIONS_VARIANT", variant) + monkeypatch.setattr( + "commons.service.hook_client.on_session_start", lambda *_args, **_kwargs: TASK_ID + ) + payload = {"session_id": str(uuid.uuid4()), "source": "startup"} + stdout = io.StringIO() + monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) + monkeypatch.setattr("sys.stdout", stdout) + assert main(["hook", "session-start"]) == 0 + context: str = json.loads(stdout.getvalue())["hookSpecificOutput"]["additionalContext"] + return context + + +@pytest.mark.parametrize("variant", ["store-line", "lesson-store"]) +@pytest.mark.parametrize( + ("setup", "store"), + [(local, "local database {db} on localhost"), (hosted, "hosted at commons.example.com:8443")], +) +def test_the_store_line_names_the_sessions_store_after_the_shipped_start_line( + machine: str, + monkeypatch: pytest.MonkeyPatch, + variant: str, + setup: Callable[[str], None], + store: str, +) -> None: + setup(machine) + monkeypatch.setattr("commons.service.hook_client.RemoteBackend.feedback", lambda _self: None) + context = session_context(monkeypatch, variant) + shipped = agent_text.VARIANTS["baseline"].start_line + assert context.endswith(f" {shipped} Store for this session: {store.format(db=machine)}.") + no_secrets(context) + + +def test_the_shipped_session_start_names_no_store( + machine: str, monkeypatch: pytest.MonkeyPatch +) -> None: + local(machine) + context = session_context(monkeypatch, "baseline") + assert "Store for this session" not in context + assert machine not in context diff --git a/server/tests/test_status.py b/server/tests/test_status.py index f608dd2..5a80565 100644 --- a/server/tests/test_status.py +++ b/server/tests/test_status.py @@ -109,7 +109,8 @@ def test_status_command_prints_a_summary( monkeypatch.setenv("COMMONS_CONFIG", str(tmp_path / "config.json")) assert main(["status"]) == 0 out = capsys.readouterr().out - assert out.startswith("entries: ") + assert out.startswith("store: ") # which store, then its figures (test_active_store.py) + assert "\nentries: " in out assert "awaiting human review: " in out assert "last 30 days: " in out diff --git a/server/tests/test_variants.py b/server/tests/test_variants.py index 076555b..aeac0fe 100644 --- a/server/tests/test_variants.py +++ b/server/tests/test_variants.py @@ -19,6 +19,7 @@ import pytest from fastmcp import FastMCP from tools import agent_eval +from tools.eval import runner from tools.eval.holdout import HOLDOUT, RETIRED, WHENEVER_IGNORE_DST from tools.eval.model import Scenario from tools.eval.scenarios import SCENARIOS @@ -395,3 +396,43 @@ def test_no_draft_ships() -> None: ): assert text not in shipped.instructions assert shipped.propose != agent_text.PROPOSE_EXAMPLES_APP + + +# Store visibility (`MIT-653`), unshipped: `store-line` adds the session's store to the shipped +# SessionStart line, and `lesson-store` is the eval condition pairing it with +# `lesson-also-propose`. The hook fills the store in (test_active_store.py). +def test_the_store_line_draft_changes_only_the_session_start_line() -> None: + shipped = agent_text.VARIANTS["baseline"].start_line + assert differing("store-line") == {"start_line"} + assert agent_text.VARIANTS["store-line"].start_line == f"{shipped} {agent_text.STORE_LINE}" + assert "{store}" in agent_text.STORE_LINE + assert "`" not in agent_text.STORE_LINE # it names the store and asks for no call + + +def test_the_lesson_store_condition_pairs_the_lesson_line_with_the_store_line() -> None: + condition = agent_text.VARIANTS["lesson-store"] + assert differing("lesson-store") == {"instructions", "start_line"} + assert condition.instructions == agent_text.VARIANTS["lesson-also-propose"].instructions + assert condition.start_line == agent_text.VARIANTS["store-line"].start_line + assert agent_text.resolve("lesson-store") == condition # what `run --variant` checks first + + +def test_the_store_line_does_not_ship() -> None: + shipped = agent_text.VARIANTS["baseline"] + assert agent_text.variant() == shipped + assert shipped.start_line is not None + assert agent_text.STORE_LINE not in shipped.start_line + assert {texts[0] for texts in agent_text.RELEASED_HOOK_TEXTS} == {agent_text.SESSION_START} + + +def test_an_eval_database_name_gives_nothing_away() -> None: + """The store line shows the agent its database's name: no scenario, condition or "eval".""" + scenario = SCENARIOS["config-app-overwrites"] + names = { + runner.database_name(condition, runner.Job(scenario, index)) + for condition in ("d-lesson-store-sonnet", "d-baseline-sonnet") + for index in (1, 2) + } + assert len(names) == 4 + for name in names: + assert re.fullmatch(r"commons_[0-9a-f]{16}", name), name diff --git a/tools/eval/runner.py b/tools/eval/runner.py index 05811fe..dd85c66 100644 --- a/tools/eval/runner.py +++ b/tools/eval/runner.py @@ -1023,9 +1023,13 @@ class Job: def database_name(condition: str, job: Job) -> str: - """Return a job's database name: unique per condition and index.""" - # Condition and index first: a long scenario name may be cut at Postgres's 63 characters. - return f"commons_eval_{condition}_{job.index}_{job.scenario.name}".replace("-", "_")[:63] + """Return a job's database name: unique per condition and index, and opaque. + + The `store-line` variants show the agent its store's database name, so the name must not + give away the scenario, the condition, or that the session is an eval. + """ + key = f"{condition}\0{job.index}\0{job.scenario.name}" + return f"commons_{hashlib.sha256(key.encode()).hexdigest()[:16]}" def grade(session: Session, events: list[dict[str, Any]]) -> Result: