Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/decisions/0036-used-detection-v1.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,9 @@ Implementation notes (T46, 2026-10-31):
- Known limit: run times are the store's clock and `matched_at` the client's. A client clock
running ahead by more than the gap between an edit and the next test run drops that run's
credit. Sending the client's run time with each test run would fix it; not done yet.
- Amended 2026-11-11 (MIT-680): fixed as above. Hooks stamp each test run with this machine's
time (`test_run_at` beside `test_run` on the wire, since older servers accept exactly
`{command, exit_code}` there), stored as `at` in the event, and pairing compares it with
`matched_at`. Runs without it (older hooks) fall back to the store's clock. The limit showed
up in tests: Postgres in Docker Desktop runs about a millisecond behind the Mac.

7 changes: 6 additions & 1 deletion server/src/commons/domain/outcomes.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,16 @@ def merge_report(existing: OutcomeRecord | None, report: Report) -> OutcomeRecor

@dataclass(frozen=True)
class ObservedRun:
"""A test command a hook saw, and how it exited."""
"""A test command a hook saw, and how it exited.

`at` is the store's clock. `client_at` is the hook's, the clock `matched_at` uses; hooks
that don't send it leave it None.
"""

command: str
exit_code: int
at: datetime
client_at: datetime | None = None


def merge_observation(existing: OutcomeRecord | None, run: ObservedRun | None) -> OutcomeRecord:
Expand Down
14 changes: 12 additions & 2 deletions server/src/commons/service/hook_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from collections.abc import Callable
from contextlib import AbstractContextManager
from dataclasses import asdict, dataclass, replace
from datetime import datetime, timedelta
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, Protocol
from uuid import UUID
Expand Down Expand Up @@ -189,6 +189,11 @@ def tool_use(
"test_run": test_run,
"file_change": file_change,
}
if test_run is not None and "at" in test_run:
# Older servers accept exactly {command, exit_code} in test_run, so the run's
# time goes beside it, where they don't look.
body["test_run"] = {k: v for k, v in test_run.items() if k != "at"}
body["test_run_at"] = test_run["at"]
if doc_lookup is not None: # servers from before MIT-622 never see the key
if bound_privacy_mode(session_id) != "full_trace": # the query stays here (0016)
doc_lookup = replace(doc_lookup, query=None)
Expand Down Expand Up @@ -321,7 +326,12 @@ def on_tool_use(backend: HookBackend, use: ToolUse) -> None:
use.command, use.exit_code, use.output or "", configured_test_runners()
)
if code is not None:
test_run = {"command": scrub(command_tail(use.command)), "exit_code": code}
test_run = {
"command": scrub(command_tail(use.command)),
"exit_code": code,
# This machine's clock, the one `matched_at` uses: the store's may differ.
"at": datetime.now(UTC).isoformat(),
}
file_change: dict[str, object] | None = None
if use.file_path is not None and use.new_text is not None:
file_change = {
Expand Down
6 changes: 4 additions & 2 deletions server/src/commons/service/hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,11 +191,13 @@ def _follows_matched_work(run: ObservedRun, seen: UseObservation) -> bool:
"""Whether a passing run can be credited to the entry: it came after work that matched.

A pass with no matched work proves nothing about the entry (decision 0036). Failing runs
still count, so an agent's claim of success can be checked against them.
still count, so an agent's claim of success can be checked against them. `matched_at` is
the hook's clock, so it is compared with the hook's time of the run when there is one.
"""
if seen.used not in {"yes", "partial"}:
return False
return seen.matched_at is None or run.at > seen.matched_at
ran = run.client_at or run.at # hooks from before the run's time: the store's clock
return seen.matched_at is None or ran > seen.matched_at


def artifact_frequencies(conn: Connection) -> ArtifactFrequencies:
Expand Down
8 changes: 6 additions & 2 deletions server/src/commons/store/task_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,15 @@ def add_task_event(
def test_runs(conn: Connection, task_id: UUID) -> list[ObservedRun]:
"""The task's test runs, oldest first."""
rows = conn.execute(
"SELECT data->>'command', (data->>'exit_code')::int, created_at FROM task_events "
"SELECT data->>'command', (data->>'exit_code')::int, created_at, "
"(data->>'at')::timestamptz FROM task_events "
"WHERE task_id = %s AND kind = 'test_run' ORDER BY created_at",
(task_id,),
).fetchall()
return [ObservedRun(command=command, exit_code=code, at=at) for command, code, at in rows]
return [
ObservedRun(command=command, exit_code=code, at=at, client_at=client_at)
for command, code, at, client_at in rows
]


@dataclass(frozen=True)
Expand Down
31 changes: 28 additions & 3 deletions server/src/commons/web/hooks_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import uuid
from collections.abc import Awaitable, Callable
from contextlib import AbstractContextManager
from datetime import datetime
from datetime import UTC, datetime
from http import HTTPStatus
from typing import Any
from uuid import UUID
Expand Down Expand Up @@ -199,14 +199,39 @@ def _tool_use(conn: Connection, body: Body) -> Body:
hooks.record_tool_use(
conn,
_session_id(body),
_test_run(body.get("test_run")),
_test_run(body.get("test_run"), body.get("test_run_at")),
_file_change(body.get("file_change")),
_doc_lookup(body.get("doc_lookup")),
)
return {}


def _test_run(value: object) -> dict[str, object] | None:
def _test_run(value: object, at: object) -> dict[str, object] | None:
"""Return the run to store, with the client's time of it when sent (absent before MIT-680)."""
run = _run_shape(value)
if run is None or at is None:
return run
return {**run, "at": _aware_time(at, "test_run_at").isoformat()}


def _aware_time(value: object, name: str) -> datetime:
"""Return the time in UTC, which any offset Python accepts converts to and Postgres reads.

Postgres can't cast an offset with fractional seconds, and a stored one would break every
later Stop for the task.
"""
try:
parsed = datetime.fromisoformat(value) if isinstance(value, str) else None
utc = None if parsed is None or parsed.tzinfo is None else parsed.astimezone(UTC)
except (ValueError, OverflowError): # OverflowError: a year edge shifted out of range
utc = None
if utc is None:
msg = f"{name} must be an ISO time with a UTC offset."
raise InvalidInputError(msg)
return utc


def _run_shape(value: object) -> dict[str, object] | None:
if value is None:
return None
if not (
Expand Down
10 changes: 9 additions & 1 deletion server/tests/test_hook_observe.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import json
import stat
import uuid
from datetime import datetime
from pathlib import Path
from typing import Any

Expand Down Expand Up @@ -59,7 +60,14 @@ def events(database_url: str, task_id: str) -> list[tuple[str, dict[str, Any]]]:
"SELECT kind::text, data FROM task_events WHERE task_id = %s ORDER BY created_at",
(task_id,),
).fetchall()
return [(kind, data) for kind, data in rows]
return [(kind, untimed(kind, data)) for kind, data in rows]


def untimed(kind: str, data: dict[str, Any]) -> dict[str, Any]:
"""The event without the hook's time of a test run, which every run must carry."""
if kind == "test_run":
assert datetime.fromisoformat(data.pop("at")).tzinfo is not None
return data


def spool(session_id: str) -> list[dict[str, Any]]:
Expand Down
97 changes: 95 additions & 2 deletions server/tests/test_hosted_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import replace
from datetime import datetime, timedelta
from datetime import datetime, timedelta, tzinfo
from pathlib import Path
from typing import Any

Expand All @@ -16,13 +16,14 @@
import pytest
from starlette.applications import Starlette
from starlette.testclient import TestClient
from typing_extensions import override

import factories
from commons.cli.main import main
from commons.domain.keys import KeyOptions
from commons.domain.observations import DocLookup, ToolUse
from commons.domain.search import SearchRequest
from commons.service import agent_text, hook_client
from commons.service import agent_text, hook_client, local_state
from commons.service import search as search_service
from commons.service.hook_client import RemoteBackend
from commons.service.identity import acting_as
Expand Down Expand Up @@ -103,6 +104,65 @@ def test_a_session_end_to_end_over_http(app: Starlette, db: psycopg.Connection)
assert "markupsafe<2.1\npip check" in [r.get("text") for r in read_spool(session)]


class ClockAhead(datetime):
"""This machine's clock, running a minute ahead of the store's."""

@override
@classmethod
def now(cls, tz: tzinfo | None = None) -> "ClockAhead":
return super().now(tz) + timedelta(minutes=1)


@pytest.mark.usefixtures("state")
def test_a_pass_after_the_fix_counts_when_this_machines_clock_runs_ahead(
app: Starlette, db: psycopg.Connection, monkeypatch: pytest.MonkeyPatch
) -> None:
"""The pass and the matched edit are timed on one clock, not the store's and this one's.

Postgres in Docker runs a little behind the Mac, which once dropped the end-to-end
test's credit; a hosted server's clock can be off by seconds.
"""
monkeypatch.setattr(local_state, "datetime", ClockAhead)
monkeypatch.setattr(hook_client, "datetime", ClockAhead)
key = create_key(db, "ahead-dev", admin=False)
backend = RemoteBackend(client(app, key))
session = str(uuid.uuid4())
task_id = hook_client.on_session_start(backend, session, None, HOOK_VERSION)
reporter = find_reporter(db, "ahead-dev")
assert reporter is not None
word = f"ahead{uuid.uuid4().hex[:8]}"
factories.entry(db, reporter, symptom=word, fix=FIX)
with acting_as(reporter):
search_service.search(db, SearchRequest(word, task_id, None, 5), "claude-x")

hook_client.on_tool_use(backend, edit(session, "markupsafe<2.1\npip check"))
hook_client.on_tool_use(backend, bash(session, "pytest -q", 0))
hook_client.on_stop(backend, session, None)

row = db.execute(
"SELECT o.used::text, o.outcome::text FROM outcomes o JOIN retrievals r "
"ON r.id = o.retrieval_id WHERE r.task_id = %s",
(task_id,),
).fetchone()
assert row == ("yes", "tests_passed")


@pytest.mark.usefixtures("state")
def test_older_servers_still_accept_the_test_run(app: Starlette, db: psycopg.Connection) -> None:
"""The run's time travels beside `test_run`, whose keys older servers check exactly."""
sent: list[dict[str, Any]] = []
http = client(app, create_key(db, "compat-dev", admin=False))
http.event_hooks = {"request": [lambda request: sent.append(json.loads(request.content))]}
session = str(uuid.uuid4())
hook_client.on_session_start(RemoteBackend(http), session, None, HOOK_VERSION)

hook_client.on_tool_use(RemoteBackend(http), bash(session, "pytest -q", 0))

body = sent[-1]
assert set(body["test_run"]) == {"command", "exit_code"}
assert datetime.fromisoformat(body["test_run_at"]).tzinfo is not None


def post(app: Starlette, key: str | None, step: str, body: dict[str, Any]) -> httpx2.Response:
return client(app, key).post(f"/api/hooks/{step}", json=body)

Expand Down Expand Up @@ -172,6 +232,17 @@ def test_local_privacy_reporters_are_refused(app: Starlette, db: psycopg.Connect
"body",
[
{"session_id": "s", "test_run": {"command": "pytest", "exit_code": 0, "stdout": "code"}},
{"session_id": "s", "test_run": {"command": "pytest", "exit_code": 0}, "test_run_at": 5},
{
"session_id": "s",
"test_run": {"command": "pytest", "exit_code": 0},
"test_run_at": "2026-11-11T10:00:00", # no zone: can't be compared
},
{
"session_id": "s",
"test_run": {"command": "pytest", "exit_code": 0},
"test_run_at": "0001-01-01T00:00:00+01:00", # out of range in UTC
},
{"session_id": "s", "file_change": {"path_sha256": "x", "content_sha256": "y"}},
{"session_id": "s", "file_change": {"content": "def secret(): ..."}},
{
Expand Down Expand Up @@ -203,6 +274,28 @@ def test_only_derived_shapes_are_accepted(
assert response.json()["error"]["code"] == "invalid_input"


def test_a_run_time_with_any_offset_is_stored_in_utc(
app: Starlette, db: psycopg.Connection
) -> None:
"""Postgres can't cast a fractional-second offset; stored, it would break every Stop."""
key = create_key(db, "offset-dev", admin=False)
session = str(uuid.uuid4())
started = post(app, key, "session-start", {"session_id": session}).json()
body = {
"session_id": session,
"test_run": {"command": "pytest", "exit_code": 0},
"test_run_at": "2026-11-11T10:00:00+05:00:30.5",
}
assert post(app, key, "tool-use", body).status_code == 200

(stored,) = db.execute(
"SELECT data->>'at' FROM task_events WHERE task_id = %s", (started["task_id"],)
).fetchone() or (None,)
assert stored == "2026-11-11T04:59:29.500000+00:00"
stop = {"session_id": session, "used": {}, "matched_at": {}}
assert post(app, key, "stop", stop).status_code == 200


def doc_lookups(db: psycopg.Connection, task_id: uuid.UUID) -> list[tuple[Any, ...]]:
return db.execute(
"SELECT library_id, version, query, privacy_mode::text, excluded FROM doc_lookups "
Expand Down
Loading