Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions docs/eval/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -825,6 +825,37 @@ two known-entry scenarios (and no duplicate proposal there), searches and propos
controls, rejected `propose` calls, and fix rate. The bar is decision 0023's. MIT-598 holds the
go/no-go.

## Drafted variant: the session's store (MIT-653, 2026-11-11)

In two real misses the agent used the wrong store, or wrote the lesson to AGENTS.md instead of
the commons. Nothing ships: `baseline` and the hook texts are unchanged, and the store line's
hook version joins `RELEASED_HOOK_TEXTS` only if it ships.

| Variant | Surface | Change |
|---|---|---|
| `store-line` | SessionStart line | The shipped line plus "Store for this session: {store}.", the hook filling in `hosted at <host>` or `local database <name> on <host>` (`identity.store_label`: no key, no password) |
| `lesson-store` | instructions and SessionStart line | `lesson-also-propose` and `store-line` together: the condition to run |

Eval sessions run on a fresh local database each, and the store line shows the agent its name,
so `runner.database_name` is now an opaque hash (`commons_<16 hex>`): the old name spelled out
the condition and scenario. A `--store NAME` experiment store is shown under the name you give it.

Not run yet (the owner starts each batch). On the four no-test scenarios and the five genuine
controls, 3 runs each, on both models: 9 scenarios x 3 runs x 2 conditions x 2 models =
**108 sessions**.

```
scenarios=(--scenario config-app-overwrites --scenario config-default-wins
--scenario known-config-app-overwrites --scenario known-config-default-wins
--scenario control-typo --scenario control-arithmetic --scenario control-clear-error
--scenario control-documented-option --scenario control-documented-flag --runs 3)
for model in sonnet claude-opus-5-5; do
uv run python tools/agent_eval.py run "${scenarios[@]}" --model $model --condition st-baseline-$model
uv run python tools/agent_eval.py run "${scenarios[@]}" --model $model --variant lesson-store --condition st-lesson-store-$model
done
uv run python tools/agent_eval.py report --condition st-baseline-sonnet --condition st-lesson-store-sonnet
```

## Hold-out change: whenever retired, Polars added (2026-11-08)

**What changed.** `real-whenever-ignore-dst` is out of the measured hold-out set, and
Expand Down
13 changes: 11 additions & 2 deletions server/src/commons/cli/hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from commons.service import agent_text, hook_client
from commons.service.hook_client import HookBackend, LocalBackend, RemoteBackend, remote_client
from commons.service.hooks import StopResult
from commons.service.identity import hosted_server
from commons.service.identity import hosted_server, store_label
from commons.service.local_state import bind_store, bound_store, state_dir
from commons.store.connection import connect

Expand Down Expand Up @@ -43,7 +43,7 @@ def _session_start(payload: Payload) -> HookOutput:
context = agent_text.SESSION_START.format(task_id=task_id)
start_line = shown.start_line
if start_line:
context = f"{context} {start_line}"
context = f"{context} {start_line.format(store=_session_store(session_id))}"
if feedback := backend.feedback(): # off unless the store turns it on (decision 0035)
context = f"{context} {feedback}"
return {"hookSpecificOutput": {"hookEventName": "SessionStart", "additionalContext": context}}
Expand All @@ -63,6 +63,15 @@ def _observe(use: ToolUse) -> None:
hook_client.on_tool_use(backend_for(use.session_id), use)


def _session_store(session_id: str) -> str:
"""Name the store this session's hooks use, as `backend_for` picks it, without secrets."""
bound = bound_store(session_id)
if bound is None:
current = _current_server()
return store_label(None if current is None else current[0])
return store_label(None if bound["store"] == "local" else bound["store"])


def _current_server() -> tuple[str, str] | None:
try:
return hosted_server()
Expand Down
8 changes: 8 additions & 0 deletions server/src/commons/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,11 @@
from commons.service.hook_install import default_settings_path, install_hooks
from commons.service.identity import (
acting_as,
active_store,
hosted_server_or_none,
init_reporter,
save_server,
store_label,
)
from commons.store import insights
from commons.store.connection import connect
Expand Down Expand Up @@ -131,6 +133,12 @@ def _pending(_args: argparse.Namespace) -> int:


def _status(_args: argparse.Namespace) -> int:
# The store goes first, before the database is reached, so it shows even when that fails.
store, local = active_store(), store_label(None)
unset = "none configured; run `commons init` (local) or `commons join` (hosted)"
sys.stdout.write(f"store: {store or unset}\n")
if store != local: # the statistics are always the local database's
sys.stdout.write(f"statistics from: {local}\n")
with connect() as conn:
result = status_service.status(conn)
counts = ", ".join(f"{count} {status}" for status, count in result.entries_by_status.items())
Expand Down
12 changes: 12 additions & 0 deletions server/src/commons/service/agent_text.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,18 @@ def _draft(edits: list[tuple[str, str]], propose: str) -> Variant:
VARIANTS["propose-example"] = _draft([], PROPOSE_EXAMPLES_APP)
VARIANTS["combined"] = _draft(list(_DRAFT_EDITS.values()), PROPOSE_EXAMPLES_APP)

# Store visibility (MIT-653), not shipped: in two real misses the agent used the wrong store,
# or wrote the lesson to AGENTS.md instead of the commons. The SessionStart context names the
# session's store after the shipped start line; the hook fills `{store}` with
# `identity.store_label` (a hosted server's host, or the local database's name, never a secret).
# Shipping it changes what installed hooks show, so its hook version joins RELEASED_HOOK_TEXTS
# only then.
STORE_LINE = "Store for this session: {store}."
_STORE_START_LINE = f"{VARIANTS['baseline'].start_line} {STORE_LINE}"
VARIANTS["store-line"] = replace(VARIANTS["baseline"], start_line=_STORE_START_LINE)
# The eval condition that pairs it with `lesson-also-propose`: both aim at the AGENTS.md miss.
VARIANTS["lesson-store"] = replace(VARIANTS["lesson-also-propose"], start_line=_STORE_START_LINE)


def _variant_name() -> str:
return os.environ.get("COMMONS_INSTRUCTIONS_VARIANT", "baseline")
Expand Down
25 changes: 21 additions & 4 deletions server/src/commons/service/doctor.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,15 @@
NotPermittedError,
StoreUnavailableError,
)
from commons.privacy.redaction import scrub
from commons.service.hook_install import default_settings_path, is_commons_hook
from commons.service.identity import config_path, current_reporter, hosted_server, read_config
from commons.service.identity import (
active_store,
config_path,
current_reporter,
hosted_server,
read_config,
)
from commons.service.local_state import active_sessions, state_dir
from commons.store.connection import Connection
from commons.store.migrate import pending_migrations
Expand All @@ -42,7 +49,7 @@ class Check:

def run_checks(connect: Connect, ping: Ping) -> list[Check]:
"""Run every check; a failure in one doesn't stop the others."""
checks = [_config()]
checks = [_config(), _store()]
server = _hosted()
if server is None:
checks += _local_database(connect)
Expand All @@ -63,6 +70,16 @@ def _config() -> Check:
return Check("config", "ok", f"{path} ({mode} mode)")


def _store() -> Check:
"""Name the store searches, proposals and hooks go to, so a wrong one is plain to see."""
store = active_store()
if store is None:
return Check(
"store", "warn", "none until `commons init` (local) or `commons join` (hosted)"
)
return Check("store", "ok", store)


def _hosted() -> tuple[str, str] | None:
try:
return hosted_server()
Expand Down Expand Up @@ -96,8 +113,8 @@ def _server(server: tuple[str, str], ping: Ping) -> Check:
try:
ping(url, key)
except (NotInitializedError, NotPermittedError, httpx2.HTTPError) as error:
return Check("server", "fail", f"{url}: {error}")
return Check("server", "ok", f"{url} accepts your key")
return Check("server", "fail", scrub(f"{url}: {error}")) # a URL may carry a password
return Check("server", "ok", scrub(f"{url} accepts your key"))


def _hooks() -> Check:
Expand Down
25 changes: 24 additions & 1 deletion server/src/commons/service/identity.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@
from contextvars import ContextVar
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlsplit
from uuid import UUID

from commons.domain.errors import NotInitializedError, NotPermittedError
from commons.domain.reporters import PrivacyDefault, validate_pseudonym
from commons.store.connection import Connection
from commons.privacy.redaction import scrub
from commons.store.connection import Connection, database_label
from commons.store.models import model_id
from commons.store.reporters import find_reporter, pseudonym_of, upsert_reporter

Expand Down Expand Up @@ -191,3 +193,24 @@ def hosted_server_or_none() -> tuple[str, str] | None:
return hosted_server()
except NotInitializedError:
return None


def store_label(server: str | None) -> str:
"""Name a store without its secrets: `hosted at <host>` or `local database <name> on <host>`.

`server` is a hosted server's URL, or None for the local database. Neither the API key nor
a password in either URL is shown, and the result goes through `redact` as well.
"""
if server is None:
return scrub(f"local database {database_label()}")
address = urlsplit(server).netloc.rpartition("@")[2] or server
return scrub(f"hosted at {address}")


def active_store() -> str | None:
"""Name the store this machine's commands and new sessions use; None without a config."""
try:
server = hosted_server()
except NotInitializedError:
return None
return store_label(None if server is None else server[0])
9 changes: 9 additions & 0 deletions server/src/commons/store/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@ def database_url() -> str:
return os.environ.get("COMMONS_DATABASE_URL", DEFAULT_DATABASE_URL)


def database_label() -> str:
"""Name the configured database as `<name> on <host>`, never with its user or password."""
try:
params = conninfo_to_dict(database_url())
except psycopg.ProgrammingError:
return "from an unreadable COMMONS_DATABASE_URL"
return f"{params.get('dbname', '?')} on {params.get('host', '?')}"


def connect() -> Connection:
"""Open a connection to the configured database, or raise StoreUnavailableError."""
try:
Expand Down
Loading
Loading