From d6e9cabea1e292d8f1978576e70868f4c925db54 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Wed, 11 Nov 2026 17:33:45 -0500 Subject: [PATCH 1/2] fix(hooks): pair a test pass with matched work on the hook's clock matched_at is this machine's clock and a run's time was the store's (clock_timestamp()), so a store clock behind this one dropped the pass's credit and left the outcome unknown. Postgres in Docker Desktop runs about a millisecond behind the Mac, which failed the hosted end-to-end test once. Hooks now stamp each test run with their own time, sent as test_run_at beside test_run (older servers check test_run's keys exactly), and pairing compares that with matched_at. Runs from older hooks keep the store's time. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- docs/decisions/0036-used-detection-v1.md | 5 ++ server/src/commons/domain/outcomes.py | 7 ++- server/src/commons/service/hook_client.py | 14 ++++- server/src/commons/service/hooks.py | 6 +- server/src/commons/store/task_events.py | 8 ++- server/src/commons/web/hooks_api.py | 23 +++++++- server/tests/test_hook_observe.py | 10 +++- server/tests/test_hosted_hooks.py | 70 ++++++++++++++++++++++- 8 files changed, 131 insertions(+), 12 deletions(-) diff --git a/docs/decisions/0036-used-detection-v1.md b/docs/decisions/0036-used-detection-v1.md index 31c8b15..706e25e 100644 --- a/docs/decisions/0036-used-detection-v1.md +++ b/docs/decisions/0036-used-detection-v1.md @@ -66,4 +66,9 @@ Implementation notes (T46, 2026-10-31): - Known limit: run times are the store's clock and `matched_at` the client's. A client clock running ahead by more than the gap between an edit and the next test run drops that run's credit. Sending the client's run time with each test run would fix it; not done yet. +- Amended 2026-11-11 (MIT-680): fixed as above. Hooks stamp each test run with this machine's + time (`test_run_at` beside `test_run` on the wire, since older servers accept exactly + `{command, exit_code}` there), stored as `at` in the event, and pairing compares it with + `matched_at`. Runs without it (older hooks) fall back to the store's clock. The limit showed + up in tests: Postgres in Docker Desktop runs about a millisecond behind the Mac. diff --git a/server/src/commons/domain/outcomes.py b/server/src/commons/domain/outcomes.py index cdbda6a..6618fe0 100644 --- a/server/src/commons/domain/outcomes.py +++ b/server/src/commons/domain/outcomes.py @@ -82,11 +82,16 @@ def merge_report(existing: OutcomeRecord | None, report: Report) -> OutcomeRecor @dataclass(frozen=True) class ObservedRun: - """A test command a hook saw, and how it exited.""" + """A test command a hook saw, and how it exited. + + `at` is the store's clock. `client_at` is the hook's, the clock `matched_at` uses; hooks + that don't send it leave it None. + """ command: str exit_code: int at: datetime + client_at: datetime | None = None def merge_observation(existing: OutcomeRecord | None, run: ObservedRun | None) -> OutcomeRecord: diff --git a/server/src/commons/service/hook_client.py b/server/src/commons/service/hook_client.py index 81985e3..7f0c03a 100644 --- a/server/src/commons/service/hook_client.py +++ b/server/src/commons/service/hook_client.py @@ -9,7 +9,7 @@ from collections.abc import Callable from contextlib import AbstractContextManager from dataclasses import asdict, dataclass, replace -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, Protocol from uuid import UUID @@ -189,6 +189,11 @@ def tool_use( "test_run": test_run, "file_change": file_change, } + if test_run is not None and "at" in test_run: + # Older servers accept exactly {command, exit_code} in test_run, so the run's + # time goes beside it, where they don't look. + body["test_run"] = {k: v for k, v in test_run.items() if k != "at"} + body["test_run_at"] = test_run["at"] if doc_lookup is not None: # servers from before MIT-622 never see the key if bound_privacy_mode(session_id) != "full_trace": # the query stays here (0016) doc_lookup = replace(doc_lookup, query=None) @@ -321,7 +326,12 @@ def on_tool_use(backend: HookBackend, use: ToolUse) -> None: use.command, use.exit_code, use.output or "", configured_test_runners() ) if code is not None: - test_run = {"command": scrub(command_tail(use.command)), "exit_code": code} + test_run = { + "command": scrub(command_tail(use.command)), + "exit_code": code, + # This machine's clock, the one `matched_at` uses: the store's may differ. + "at": datetime.now(UTC).isoformat(), + } file_change: dict[str, object] | None = None if use.file_path is not None and use.new_text is not None: file_change = { diff --git a/server/src/commons/service/hooks.py b/server/src/commons/service/hooks.py index aaeb2e6..6212a60 100644 --- a/server/src/commons/service/hooks.py +++ b/server/src/commons/service/hooks.py @@ -191,11 +191,13 @@ def _follows_matched_work(run: ObservedRun, seen: UseObservation) -> bool: """Whether a passing run can be credited to the entry: it came after work that matched. A pass with no matched work proves nothing about the entry (decision 0036). Failing runs - still count, so an agent's claim of success can be checked against them. + still count, so an agent's claim of success can be checked against them. `matched_at` is + the hook's clock, so it is compared with the hook's time of the run when there is one. """ if seen.used not in {"yes", "partial"}: return False - return seen.matched_at is None or run.at > seen.matched_at + ran = run.client_at or run.at # hooks from before the run's time: the store's clock + return seen.matched_at is None or ran > seen.matched_at def artifact_frequencies(conn: Connection) -> ArtifactFrequencies: diff --git a/server/src/commons/store/task_events.py b/server/src/commons/store/task_events.py index ecc97f5..0a206d1 100644 --- a/server/src/commons/store/task_events.py +++ b/server/src/commons/store/task_events.py @@ -33,11 +33,15 @@ def add_task_event( def test_runs(conn: Connection, task_id: UUID) -> list[ObservedRun]: """The task's test runs, oldest first.""" rows = conn.execute( - "SELECT data->>'command', (data->>'exit_code')::int, created_at FROM task_events " + "SELECT data->>'command', (data->>'exit_code')::int, created_at, " + "(data->>'at')::timestamptz FROM task_events " "WHERE task_id = %s AND kind = 'test_run' ORDER BY created_at", (task_id,), ).fetchall() - return [ObservedRun(command=command, exit_code=code, at=at) for command, code, at in rows] + return [ + ObservedRun(command=command, exit_code=code, at=at, client_at=client_at) + for command, code, at, client_at in rows + ] @dataclass(frozen=True) diff --git a/server/src/commons/web/hooks_api.py b/server/src/commons/web/hooks_api.py index f9dcd8d..f15f56c 100644 --- a/server/src/commons/web/hooks_api.py +++ b/server/src/commons/web/hooks_api.py @@ -199,14 +199,33 @@ def _tool_use(conn: Connection, body: Body) -> Body: hooks.record_tool_use( conn, _session_id(body), - _test_run(body.get("test_run")), + _test_run(body.get("test_run"), body.get("test_run_at")), _file_change(body.get("file_change")), _doc_lookup(body.get("doc_lookup")), ) return {} -def _test_run(value: object) -> dict[str, object] | None: +def _test_run(value: object, at: object) -> dict[str, object] | None: + """Return the run to store, with the client's time of it when sent (absent before MIT-680).""" + run = _run_shape(value) + if run is None or at is None: + return run + return {**run, "at": _aware_time(at, "test_run_at").isoformat()} + + +def _aware_time(value: object, name: str) -> datetime: + try: + parsed = datetime.fromisoformat(value) if isinstance(value, str) else None + except ValueError: + parsed = None + if parsed is None or parsed.tzinfo is None: + msg = f"{name} must be an ISO time with a UTC offset." + raise InvalidInputError(msg) + return parsed + + +def _run_shape(value: object) -> dict[str, object] | None: if value is None: return None if not ( diff --git a/server/tests/test_hook_observe.py b/server/tests/test_hook_observe.py index 264be7f..553d66d 100644 --- a/server/tests/test_hook_observe.py +++ b/server/tests/test_hook_observe.py @@ -1,6 +1,7 @@ import json import stat import uuid +from datetime import datetime from pathlib import Path from typing import Any @@ -59,7 +60,14 @@ def events(database_url: str, task_id: str) -> list[tuple[str, dict[str, Any]]]: "SELECT kind::text, data FROM task_events WHERE task_id = %s ORDER BY created_at", (task_id,), ).fetchall() - return [(kind, data) for kind, data in rows] + return [(kind, untimed(kind, data)) for kind, data in rows] + + +def untimed(kind: str, data: dict[str, Any]) -> dict[str, Any]: + """The event without the hook's time of a test run, which every run must carry.""" + if kind == "test_run": + assert datetime.fromisoformat(data.pop("at")).tzinfo is not None + return data def spool(session_id: str) -> list[dict[str, Any]]: diff --git a/server/tests/test_hosted_hooks.py b/server/tests/test_hosted_hooks.py index cb6b8fb..bbcd354 100644 --- a/server/tests/test_hosted_hooks.py +++ b/server/tests/test_hosted_hooks.py @@ -7,7 +7,7 @@ from collections.abc import Iterator from contextlib import contextmanager from dataclasses import replace -from datetime import datetime, timedelta +from datetime import datetime, timedelta, tzinfo from pathlib import Path from typing import Any @@ -16,13 +16,14 @@ import pytest from starlette.applications import Starlette from starlette.testclient import TestClient +from typing_extensions import override import factories from commons.cli.main import main from commons.domain.keys import KeyOptions from commons.domain.observations import DocLookup, ToolUse from commons.domain.search import SearchRequest -from commons.service import agent_text, hook_client +from commons.service import agent_text, hook_client, local_state from commons.service import search as search_service from commons.service.hook_client import RemoteBackend from commons.service.identity import acting_as @@ -103,6 +104,65 @@ def test_a_session_end_to_end_over_http(app: Starlette, db: psycopg.Connection) assert "markupsafe<2.1\npip check" in [r.get("text") for r in read_spool(session)] +class ClockAhead(datetime): + """This machine's clock, running a minute ahead of the store's.""" + + @override + @classmethod + def now(cls, tz: tzinfo | None = None) -> "ClockAhead": + return super().now(tz) + timedelta(minutes=1) + + +@pytest.mark.usefixtures("state") +def test_a_pass_after_the_fix_counts_when_this_machines_clock_runs_ahead( + app: Starlette, db: psycopg.Connection, monkeypatch: pytest.MonkeyPatch +) -> None: + """The pass and the matched edit are timed on one clock, not the store's and this one's. + + Postgres in Docker runs a little behind the Mac, which once dropped the end-to-end + test's credit; a hosted server's clock can be off by seconds. + """ + monkeypatch.setattr(local_state, "datetime", ClockAhead) + monkeypatch.setattr(hook_client, "datetime", ClockAhead) + key = create_key(db, "ahead-dev", admin=False) + backend = RemoteBackend(client(app, key)) + session = str(uuid.uuid4()) + task_id = hook_client.on_session_start(backend, session, None, HOOK_VERSION) + reporter = find_reporter(db, "ahead-dev") + assert reporter is not None + word = f"ahead{uuid.uuid4().hex[:8]}" + factories.entry(db, reporter, symptom=word, fix=FIX) + with acting_as(reporter): + search_service.search(db, SearchRequest(word, task_id, None, 5), "claude-x") + + hook_client.on_tool_use(backend, edit(session, "markupsafe<2.1\npip check")) + hook_client.on_tool_use(backend, bash(session, "pytest -q", 0)) + hook_client.on_stop(backend, session, None) + + row = db.execute( + "SELECT o.used::text, o.outcome::text FROM outcomes o JOIN retrievals r " + "ON r.id = o.retrieval_id WHERE r.task_id = %s", + (task_id,), + ).fetchone() + assert row == ("yes", "tests_passed") + + +@pytest.mark.usefixtures("state") +def test_older_servers_still_accept_the_test_run(app: Starlette, db: psycopg.Connection) -> None: + """The run's time travels beside `test_run`, whose keys older servers check exactly.""" + sent: list[dict[str, Any]] = [] + http = client(app, create_key(db, "compat-dev", admin=False)) + http.event_hooks = {"request": [lambda request: sent.append(json.loads(request.content))]} + session = str(uuid.uuid4()) + hook_client.on_session_start(RemoteBackend(http), session, None, HOOK_VERSION) + + hook_client.on_tool_use(RemoteBackend(http), bash(session, "pytest -q", 0)) + + body = sent[-1] + assert set(body["test_run"]) == {"command", "exit_code"} + assert datetime.fromisoformat(body["test_run_at"]).tzinfo is not None + + def post(app: Starlette, key: str | None, step: str, body: dict[str, Any]) -> httpx2.Response: return client(app, key).post(f"/api/hooks/{step}", json=body) @@ -172,6 +232,12 @@ def test_local_privacy_reporters_are_refused(app: Starlette, db: psycopg.Connect "body", [ {"session_id": "s", "test_run": {"command": "pytest", "exit_code": 0, "stdout": "code"}}, + {"session_id": "s", "test_run": {"command": "pytest", "exit_code": 0}, "test_run_at": 5}, + { + "session_id": "s", + "test_run": {"command": "pytest", "exit_code": 0}, + "test_run_at": "2026-11-11T10:00:00", # no zone: can't be compared + }, {"session_id": "s", "file_change": {"path_sha256": "x", "content_sha256": "y"}}, {"session_id": "s", "file_change": {"content": "def secret(): ..."}}, { From 3fa05c140c130ecb6c605db28ac5a3476f77cdb0 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Wed, 11 Nov 2026 17:42:53 -0500 Subject: [PATCH 2/2] fix(hooks): store a test run's client time in UTC test_run_at kept the client's offset. Python accepts offsets with fractional seconds (+05:00:30.5) that Postgres can't cast to timestamptz, so a stored one would fail every later Stop for the task. Normalize to UTC before storing, and refuse times that overflow when shifted to UTC at the year edges (0001-01-01T00:00:00+01:00) with a 400. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- server/src/commons/web/hooks_api.py | 16 +++++++++++----- server/tests/test_hosted_hooks.py | 27 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/server/src/commons/web/hooks_api.py b/server/src/commons/web/hooks_api.py index f15f56c..4a9f4a0 100644 --- a/server/src/commons/web/hooks_api.py +++ b/server/src/commons/web/hooks_api.py @@ -10,7 +10,7 @@ import uuid from collections.abc import Awaitable, Callable from contextlib import AbstractContextManager -from datetime import datetime +from datetime import UTC, datetime from http import HTTPStatus from typing import Any from uuid import UUID @@ -215,14 +215,20 @@ def _test_run(value: object, at: object) -> dict[str, object] | None: def _aware_time(value: object, name: str) -> datetime: + """Return the time in UTC, which any offset Python accepts converts to and Postgres reads. + + Postgres can't cast an offset with fractional seconds, and a stored one would break every + later Stop for the task. + """ try: parsed = datetime.fromisoformat(value) if isinstance(value, str) else None - except ValueError: - parsed = None - if parsed is None or parsed.tzinfo is None: + utc = None if parsed is None or parsed.tzinfo is None else parsed.astimezone(UTC) + except (ValueError, OverflowError): # OverflowError: a year edge shifted out of range + utc = None + if utc is None: msg = f"{name} must be an ISO time with a UTC offset." raise InvalidInputError(msg) - return parsed + return utc def _run_shape(value: object) -> dict[str, object] | None: diff --git a/server/tests/test_hosted_hooks.py b/server/tests/test_hosted_hooks.py index bbcd354..07ecdf7 100644 --- a/server/tests/test_hosted_hooks.py +++ b/server/tests/test_hosted_hooks.py @@ -238,6 +238,11 @@ def test_local_privacy_reporters_are_refused(app: Starlette, db: psycopg.Connect "test_run": {"command": "pytest", "exit_code": 0}, "test_run_at": "2026-11-11T10:00:00", # no zone: can't be compared }, + { + "session_id": "s", + "test_run": {"command": "pytest", "exit_code": 0}, + "test_run_at": "0001-01-01T00:00:00+01:00", # out of range in UTC + }, {"session_id": "s", "file_change": {"path_sha256": "x", "content_sha256": "y"}}, {"session_id": "s", "file_change": {"content": "def secret(): ..."}}, { @@ -269,6 +274,28 @@ def test_only_derived_shapes_are_accepted( assert response.json()["error"]["code"] == "invalid_input" +def test_a_run_time_with_any_offset_is_stored_in_utc( + app: Starlette, db: psycopg.Connection +) -> None: + """Postgres can't cast a fractional-second offset; stored, it would break every Stop.""" + key = create_key(db, "offset-dev", admin=False) + session = str(uuid.uuid4()) + started = post(app, key, "session-start", {"session_id": session}).json() + body = { + "session_id": session, + "test_run": {"command": "pytest", "exit_code": 0}, + "test_run_at": "2026-11-11T10:00:00+05:00:30.5", + } + assert post(app, key, "tool-use", body).status_code == 200 + + (stored,) = db.execute( + "SELECT data->>'at' FROM task_events WHERE task_id = %s", (started["task_id"],) + ).fetchone() or (None,) + assert stored == "2026-11-11T04:59:29.500000+00:00" + stop = {"session_id": session, "used": {}, "matched_at": {}} + assert post(app, key, "stop", stop).status_code == 200 + + def doc_lookups(db: psycopg.Connection, task_id: uuid.UUID) -> list[tuple[Any, ...]]: return db.execute( "SELECT library_id, version, query, privacy_mode::text, excluded FROM doc_lookups "