From 7299bb0de61a4d16c7f336f95f96979517182f8b Mon Sep 17 00:00:00 2001 From: jsflax Date: Mon, 21 Sep 2026 12:05:51 -0400 Subject: [PATCH 1/8] Handle verified near-duplicate learner responses without write holds --- Tests/codex_plugin/test_host.py | 28 +++++ Tests/codex_plugin/test_receipts.py | 51 +++++++++ .../scripts/codex_learner/learner_prompt.md | 6 +- .../scripts/codex_learner/memory_proxy.py | 60 +++++++++- .../engram/scripts/codex_learner/runner.py | 8 ++ scripts/codex_learner/learner_prompt.md | 6 +- scripts/codex_learner/memory_proxy.py | 60 +++++++++- scripts/codex_learner/runner.py | 4 + scripts/tests/test_codex_learner_runner.py | 45 +++++++- scripts/tests/test_codex_memory_proxy.py | 105 ++++++++++++++++++ 10 files changed, 358 insertions(+), 15 deletions(-) diff --git a/Tests/codex_plugin/test_host.py b/Tests/codex_plugin/test_host.py index 3352027..755bd4b 100644 --- a/Tests/codex_plugin/test_host.py +++ b/Tests/codex_plugin/test_host.py @@ -325,6 +325,34 @@ def test_unsettled_forwarded_write_requires_reconciliation(self): self.assertEqual(result, 'reconciliation_required') self.assertEqual(state['reconciliation_required']['reason'], 'successful_or_unverified_write') + def test_known_no_write_conflict_failure_keeps_retry_without_reconciliation(self): + _, _, _, result, state = self.failure_with_audit([ + {'event': 'relay_started'}, {'event': 'tool_call', 'id': 1, 'tool': 'remember'}, + {'event': 'tool_result', 'id': 1, 'tool': 'remember', 'ok': True, + 'forwarded': True, 'memory_ids': [], 'write_outcome': 'not_stored_near_duplicate'}]) + self.assertEqual(result, 'failed') + self.assertNotIn('reconciliation_required', state) + + def test_unrecognized_forwarded_no_ids_response_stays_held(self): + _, _, _, result, state = self.failure_with_audit([ + {'event': 'relay_started'}, {'event': 'tool_call', 'id': 1, 'tool': 'remember'}, + {'event': 'tool_result', 'id': 1, 'tool': 'remember', 'ok': True, + 'forwarded': True, 'memory_ids': []}]) + self.assertEqual(result, 'reconciliation_required') + self.assertEqual(state['reconciliation_required']['reason'], 'successful_or_unverified_write') + + def test_conflict_plus_successful_write_failure_stays_held(self): + memory_id = '11111111-2222-4333-8444-555555555555' + _, _, _, result, state = self.failure_with_audit([ + {'event': 'relay_started'}, {'event': 'tool_call', 'id': 1, 'tool': 'remember'}, + {'event': 'tool_result', 'id': 1, 'tool': 'remember', 'ok': True, + 'forwarded': True, 'memory_ids': [], 'write_outcome': 'not_stored_near_duplicate'}, + {'event': 'tool_call', 'id': 2, 'tool': 'remember'}, + {'event': 'tool_result', 'id': 2, 'tool': 'remember', 'ok': True, + 'forwarded': True, 'memory_ids': [memory_id]}]) + self.assertEqual(result, 'reconciliation_required') + self.assertEqual(state['reconciliation_required']['memory_ids'], [memory_id]) + def test_missing_audit_after_provider_start_is_unknown_write_status(self): _, _, _, result, state = self.failure_with_audit(None) self.assertEqual(result, 'reconciliation_required') diff --git a/Tests/codex_plugin/test_receipts.py b/Tests/codex_plugin/test_receipts.py index 890736a..e65162b 100644 --- a/Tests/codex_plugin/test_receipts.py +++ b/Tests/codex_plugin/test_receipts.py @@ -66,6 +66,57 @@ def test_claimed_uuid_without_engram_receipt_is_denied(self): self.assertGreater(self.audit_result()['tool_errors'], 0) self.assertEqual(self.audit_result()['writes'], []) + def conflict(self): + return (PROXY.NEAR_DUPLICATE_PREFIX + + f'\n [id:{OTHER}] (distance: 0.123, term overlap: 90%) PRIVATE_EXISTING' + + PROXY.NEAR_DUPLICATE_SUFFIX) + + def test_near_duplicate_is_not_a_write_or_tool_error(self): + self.call(1, 'remember', {'content': 'Synthetic'}) + output, _ = self.reply(1, self.conflict()) + self.assertIs(output['result']['isError'], False) + self.assertIn(OTHER, output['result']['content'][0]['text']) + self.assertNotIn(OTHER, json.dumps(self.audit.rows)) + self.assertNotIn('PRIVATE_EXISTING', json.dumps(self.audit.rows)) + self.assertEqual(self.audit_result(), {'tool_calls': 1, 'write_calls': 0, + 'writes': [], 'tool_errors': 0}) + + def test_conflict_then_update_counts_only_the_verified_update(self): + self.policy = PROXY.Policy(self.audit, 'codex-session:SOURCE', 3, 2) + self.call(1, 'remember', {'content': 'Synthetic'}) + self.reply(1, self.conflict()) + self.call(2, 'update', {'id': OTHER, 'append': 'New finding'}) + self.reply(2, f'Updated memory (id: {OTHER})') + self.assertEqual(self.audit_result(), {'tool_calls': 2, 'write_calls': 1, + 'writes': [{'tool': 'update', 'memory_ids': [OTHER]}], 'tool_errors': 0}) + + def test_conflict_attempt_still_consumes_write_budget(self): + self.call(1, 'remember', {'content': 'Synthetic'}) + self.reply(1, self.conflict()) + forwarded, denied = self.call(2, 'remember', {'content': 'Another'}) + self.assertIsNone(forwarded) + self.assertTrue(denied['result']['isError']) + self.assertEqual(self.policy.write_calls, 2) + self.assertEqual(self.audit_result()['write_calls'], 0) + self.assertGreater(self.audit_result()['tool_errors'], 0) + + def test_malformed_no_write_receipts_fail_both_completion_and_reconciliation(self): + self.call(1, 'remember', {'content': 'Synthetic'}) + self.reply(1, self.conflict()) + valid = self.audit.rows[-1] + changes = [{'write_outcome': 'unknown'}, {'memory_ids': [OTHER]}, + {'memory_ids': None}, {'forwarded': 1}, {'forwarded': False}, + {'ok': 1}, {'ok': False}, {'tool': 'update'}, {'extra': True}] + for change in changes: + with self.subTest(change=change): + self.audit.rows[-1] = dict(valid, **change) + self.assertGreater(self.audit_result()['tool_errors'], 0) + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) + rows = [{'event': 'relay_started'}, *self.audit.rows] + (path / 'mcp-audit.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in rows)) + self.assertIsNotNone(F.RUNNER.failure_reconciliation(path, True)) + def test_update_receipt_requires_requested_exact_uuid(self): self.call(1, 'update', {'id': ID, 'content': 'Synthetic'}) self.reply(1, f'Updated memory (id: {OTHER})') diff --git a/codex/plugins/engram/scripts/codex_learner/learner_prompt.md b/codex/plugins/engram/scripts/codex_learner/learner_prompt.md index eaee58d..461aa29 100644 --- a/codex/plugins/engram/scripts/codex_learner/learner_prompt.md +++ b/codex/plugins/engram/scripts/codex_learner/learner_prompt.md @@ -27,5 +27,9 @@ memory IDs and a short non-sensitive summary. For stored, list only the primary ID in each successful "Stored memory (id: ...)" or "Updated memory (id: ...)" receipt, plus both endpoints of any explicit successful connect call. Do not include IDs mentioned in recall results, memory text, or automatic linking notes. -For no_new_memories, return an empty ID list and make no writes. The runner +A near-duplicate warning saying the new memory was NOT stored is a successful +no-write outcome, not a saved memory. Its existing IDs are context only. Review +whether an update is needed; otherwise finish with no_new_memories. Never force +or delete a duplicate, and count the rejected attempt against the write budget. +For no_new_memories, return an empty ID list and make no successful writes. The runner independently checks tool results before accepting the excerpt as processed. diff --git a/codex/plugins/engram/scripts/codex_learner/memory_proxy.py b/codex/plugins/engram/scripts/codex_learner/memory_proxy.py index 3979545..78ca44f 100644 --- a/codex/plugins/engram/scripts/codex_learner/memory_proxy.py +++ b/codex/plugins/engram/scripts/codex_learner/memory_proxy.py @@ -75,8 +75,51 @@ def rpc_error(request_id: Any, message: str) -> dict[str, Any]: return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32601, "message": message}} +# This warning and suffix are emitted only by remember's pre-storage conflict +# branch in MemoryTools+Core.swift. Match the outer native receipt, never a +# phrase or UUID quoted inside remembered content. +NO_WRITE_OUTCOME = "not_stored_near_duplicate" +NEAR_DUPLICATE_PREFIX = "⚠️ Near-duplicate memory detected. The new memory was NOT stored.\n\nExisting similar memories:" +NEAR_DUPLICATE_SUFFIX = ('\n\nTo resolve:' + '\n - Use `update(id: "UUID", ...)` to modify the existing memory' + '\n - Use `remember(..., force: true)` to keep both' + '\n - Use `forget(id: "UUID")` to remove the old one, then `remember` the new one') + + +def verified_no_write_response(tool: str, result: Any) -> bool: + """Recognize one vetted native no-write contract; unknown results stay unsafe.""" + if (tool != "remember" or not isinstance(result, dict) + or result.get("isError") is not False or "structuredContent" in result): + return False + content = result.get("content") + if not isinstance(content, list) or len(content) != 1: + return False + item = content[0] + if not isinstance(item, dict) or set(item) != {"type", "text"} or item.get("type") != "text": + return False + text = item.get("text") + return (isinstance(text, str) and text.startswith(NEAR_DUPLICATE_PREFIX) + and text.endswith(NEAR_DUPLICATE_SUFFIX)) + + +def verified_no_write_receipt(entry: Any) -> bool: + """Exact gateway metadata shared by completion and failed-run reconciliation.""" + return (isinstance(entry, dict) + and set(entry) == {"event", "id", "tool", "ok", "memory_ids", "forwarded", "write_outcome"} + and entry.get("event") == "tool_result" and valid_id(entry.get("id")) + and entry.get("tool") == "remember" and entry.get("ok") is True + and entry.get("forwarded") is True + and isinstance(entry.get("memory_ids"), list) and entry["memory_ids"] == [] + and entry.get("write_outcome") == NO_WRITE_OUTCOME) + + def verified_write_ids(tool: str, arguments: dict[str, Any], result: dict[str, Any]) -> list[str]: - texts = [item.get("text", "") for item in result.get("content", []) if isinstance(item, dict) and item.get("type") == "text" and isinstance(item.get("text"), str)] + content = result.get("content") + if (not isinstance(content, list) or len(content) != 1 + or not isinstance(content[0], dict) or content[0].get("type") != "text" + or not isinstance(content[0].get("text"), str) or "structuredContent" in result): + return [] + texts = [content[0]["text"]] for text in texts: if tool in {"remember", "update"}: prefix = "Stored" if tool == "remember" else "Updated" @@ -134,8 +177,11 @@ def __init__(self, audit: Audit, provenance: str, max_tool_calls: int, max_write self.message_count = 0 self.max_messages = max(64, max_tool_calls * 4 + 8) - def record_result(self, request_id: Any, tool: str, ok: bool, memory_ids: list[str] | None = None, *, forwarded: bool = True) -> None: - self.audit.write({"event": "tool_result", "id": request_id, "tool": tool, "ok": ok, "memory_ids": memory_ids or [], "forwarded": forwarded}) + def record_result(self, request_id: Any, tool: str, ok: bool, memory_ids: list[str] | None = None, *, forwarded: bool = True, write_outcome: str | None = None) -> None: + entry = {"event": "tool_result", "id": request_id, "tool": tool, "ok": ok, "memory_ids": memory_ids or [], "forwarded": forwarded} + if write_outcome is not None: + entry["write_outcome"] = write_outcome + self.audit.write(entry) def deny_tool(self, request_id: Any, tool: str, reason: str) -> tuple[None, dict[str, Any]]: self.record_result(request_id, tool, False, forwarded=False) @@ -234,10 +280,12 @@ def server_message(self, message: Any) -> tuple[dict[str, Any] | None, dict[str, if pending["method"] == "tools/call": tool = pending["tool"] ok = "error" not in message and isinstance(result, dict) and result.get("isError", False) is False - memory_ids = verified_write_ids(tool, pending["arguments"], result) if ok and tool in WRITE_TOOLS else [] - if tool in WRITE_TOOLS and not memory_ids: + no_write = ok and verified_no_write_response(tool, result) + memory_ids = verified_write_ids(tool, pending["arguments"], result) if ok and tool in WRITE_TOOLS and not no_write else [] + if tool in WRITE_TOOLS and not memory_ids and not no_write: ok = False - self.record_result(request_id, tool, ok, memory_ids if ok else []) + self.record_result(request_id, tool, ok, memory_ids if ok else [], + write_outcome=NO_WRITE_OUTCOME if no_write else None) if not ok and "error" not in message and isinstance(result, dict) and result.get("isError", False) is False: return tool_error(request_id, "write was not verified by an Engram receipt; do not retry blindly"), None return message, None diff --git a/codex/plugins/engram/scripts/codex_learner/runner.py b/codex/plugins/engram/scripts/codex_learner/runner.py index 36a2f96..2c61c16 100644 --- a/codex/plugins/engram/scripts/codex_learner/runner.py +++ b/codex/plugins/engram/scripts/codex_learner/runner.py @@ -24,6 +24,7 @@ import uuid from .transcript import inspect_rollout, read_excerpt +from .memory_proxy import verified_no_write_receipt from . import admission, memory_config EVENTS = {"Stop", "SubagentStop", "PreCompact", "SessionEnd"} @@ -522,6 +523,9 @@ def audit_tools(path: Path, *, allow_clean_interrupt: bool = False) -> dict: result = results.get(key, {}) if result.get("ok") is not True or result.get("tool") != tool: errors += 1 + elif "write_outcome" in result: + if not verified_no_write_receipt(result): + errors += 1 elif tool in WRITE_TOOLS: ids = result.get("memory_ids", []) if not isinstance(ids, list) or not ids or any(not isinstance(i, str) or not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", i) for i in ids): @@ -740,6 +744,10 @@ def failure_reconciliation(run_dir: Path, provider_started: bool | None) -> dict result = results.get(key) if result is not None and result.get("tool") != call["tool"]: return unknown + if result is not None and "write_outcome" in result: + if verified_no_write_receipt(result): + continue # A vetted native conflict completed without storage. + return unknown # Contradictory/unknown outcome metadata is unsafe. if result is not None and result.get("forwarded") is False and result.get("ok") is False: continue # A denied call is proved not to have reached Engram. risky = True diff --git a/scripts/codex_learner/learner_prompt.md b/scripts/codex_learner/learner_prompt.md index a110a34..5055e71 100644 --- a/scripts/codex_learner/learner_prompt.md +++ b/scripts/codex_learner/learner_prompt.md @@ -25,5 +25,9 @@ memory IDs and a short non-sensitive summary. For stored, list only the primary ID in each successful "Stored memory (id: ...)" or "Updated memory (id: ...)" receipt, plus both endpoints of any explicit successful connect call. Do not include IDs mentioned in recall results, memory text, or automatic linking notes. -For no_new_memories, return an empty ID list and make no writes. The runner +A near-duplicate warning saying the new memory was NOT stored is a successful +no-write outcome, not a saved memory. Its existing IDs are context only. Review +whether an update is needed; otherwise finish with no_new_memories. Never force +or delete a duplicate, and count the rejected attempt against the write budget. +For no_new_memories, return an empty ID list and make no successful writes. The runner independently checks tool results before accepting the excerpt as processed. diff --git a/scripts/codex_learner/memory_proxy.py b/scripts/codex_learner/memory_proxy.py index 94de104..37dbc30 100644 --- a/scripts/codex_learner/memory_proxy.py +++ b/scripts/codex_learner/memory_proxy.py @@ -75,8 +75,51 @@ def rpc_error(request_id: Any, message: str) -> dict[str, Any]: return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32601, "message": message}} +# This warning and suffix are emitted only by remember's pre-storage conflict +# branch in MemoryTools+Core.swift. Match the outer native receipt, never a +# phrase or UUID quoted inside remembered content. +NO_WRITE_OUTCOME = "not_stored_near_duplicate" +NEAR_DUPLICATE_PREFIX = "⚠️ Near-duplicate memory detected. The new memory was NOT stored.\n\nExisting similar memories:" +NEAR_DUPLICATE_SUFFIX = ('\n\nTo resolve:' + '\n - Use `update(id: "UUID", ...)` to modify the existing memory' + '\n - Use `remember(..., force: true)` to keep both' + '\n - Use `forget(id: "UUID")` to remove the old one, then `remember` the new one') + + +def verified_no_write_response(tool: str, result: Any) -> bool: + """Recognize one vetted native no-write contract; unknown results stay unsafe.""" + if (tool != "remember" or not isinstance(result, dict) + or result.get("isError") is not False or "structuredContent" in result): + return False + content = result.get("content") + if not isinstance(content, list) or len(content) != 1: + return False + item = content[0] + if not isinstance(item, dict) or set(item) != {"type", "text"} or item.get("type") != "text": + return False + text = item.get("text") + return (isinstance(text, str) and text.startswith(NEAR_DUPLICATE_PREFIX) + and text.endswith(NEAR_DUPLICATE_SUFFIX)) + + +def verified_no_write_receipt(entry: Any) -> bool: + """Exact gateway metadata shared by completion and failed-run reconciliation.""" + return (isinstance(entry, dict) + and set(entry) == {"event", "id", "tool", "ok", "memory_ids", "forwarded", "write_outcome"} + and entry.get("event") == "tool_result" and valid_id(entry.get("id")) + and entry.get("tool") == "remember" and entry.get("ok") is True + and entry.get("forwarded") is True + and isinstance(entry.get("memory_ids"), list) and entry["memory_ids"] == [] + and entry.get("write_outcome") == NO_WRITE_OUTCOME) + + def verified_write_ids(tool: str, arguments: dict[str, Any], result: dict[str, Any]) -> list[str]: - texts = [item.get("text", "") for item in result.get("content", []) if isinstance(item, dict) and item.get("type") == "text" and isinstance(item.get("text"), str)] + content = result.get("content") + if (not isinstance(content, list) or len(content) != 1 + or not isinstance(content[0], dict) or content[0].get("type") != "text" + or not isinstance(content[0].get("text"), str) or "structuredContent" in result): + return [] + texts = [content[0]["text"]] for text in texts: if tool in {"remember", "update"}: prefix = "Stored" if tool == "remember" else "Updated" @@ -104,8 +147,11 @@ def __init__(self, audit: Audit, provenance: str, max_tool_calls: int, max_write self.pending: dict[tuple[type, Any], dict[str, Any]] = {} self.used_ids: set[tuple[type, Any]] = set() - def record_result(self, request_id: Any, tool: str, ok: bool, memory_ids: list[str] | None = None) -> None: - self.audit.write({"event": "tool_result", "id": request_id, "tool": tool, "ok": ok, "memory_ids": memory_ids or []}) + def record_result(self, request_id: Any, tool: str, ok: bool, memory_ids: list[str] | None = None, *, write_outcome: str | None = None) -> None: + entry = {"event": "tool_result", "id": request_id, "tool": tool, "ok": ok, "memory_ids": memory_ids or []} + if write_outcome is not None: + entry.update(write_outcome=write_outcome, forwarded=True) + self.audit.write(entry) def deny_tool(self, request_id: Any, tool: str, reason: str) -> tuple[None, dict[str, Any]]: self.record_result(request_id, tool, False) @@ -196,10 +242,12 @@ def server_message(self, message: Any) -> tuple[dict[str, Any] | None, dict[str, if pending["method"] == "tools/call": tool = pending["tool"] ok = "error" not in message and isinstance(result, dict) and result.get("isError", False) is False - memory_ids = verified_write_ids(tool, pending["arguments"], result) if ok and tool in WRITE_TOOLS else [] - if tool in WRITE_TOOLS and not memory_ids: + no_write = ok and verified_no_write_response(tool, result) + memory_ids = verified_write_ids(tool, pending["arguments"], result) if ok and tool in WRITE_TOOLS and not no_write else [] + if tool in WRITE_TOOLS and not memory_ids and not no_write: ok = False - self.record_result(request_id, tool, ok, memory_ids if ok else []) + self.record_result(request_id, tool, ok, memory_ids if ok else [], + write_outcome=NO_WRITE_OUTCOME if no_write else None) if not ok and "error" not in message and isinstance(result, dict) and result.get("isError", False) is False: return tool_error(request_id, "write was not verified by an Engram receipt; do not retry blindly"), None return message, None diff --git a/scripts/codex_learner/runner.py b/scripts/codex_learner/runner.py index b355c01..953532d 100644 --- a/scripts/codex_learner/runner.py +++ b/scripts/codex_learner/runner.py @@ -23,6 +23,7 @@ import uuid from .transcript import inspect_rollout, read_excerpt +from .memory_proxy import verified_no_write_receipt EVENTS = {"Stop", "PreCompact", "SessionEnd"} GUARD = "ENGRAM_CODEX_LEARNER" @@ -346,6 +347,9 @@ def audit_tools(path: Path) -> dict: result = results.get(key, {}) if result.get("ok") is not True or result.get("tool") != tool: errors += 1 + elif "write_outcome" in result: + if not verified_no_write_receipt(result): + errors += 1 elif tool in WRITE_TOOLS: ids = result.get("memory_ids", []) if not isinstance(ids, list) or not ids or any(not isinstance(i, str) or not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", i) for i in ids): diff --git a/scripts/tests/test_codex_learner_runner.py b/scripts/tests/test_codex_learner_runner.py index f17c086..736af30 100644 --- a/scripts/tests/test_codex_learner_runner.py +++ b/scripts/tests/test_codex_learner_runner.py @@ -180,7 +180,7 @@ def execute(): if errors: raise errors[0] - def fake_codex(self, *, events=None, result=None, exit_code=0, delay=0, read_stdin=True, audit_events=None, write_audit=True): + def fake_codex(self, *, events=None, result=None, exit_code=0, delay=0, read_stdin=True, audit_events=None, write_audit=True, audit_rows=None): events = events if events is not None else [mcp_event(), {"type": "turn.completed"}] audit_events = events if audit_events is None else audit_events result = result if result is not None else {"outcome": "no_new_memories", "summary": "Already represented.", "memory_ids": []} @@ -215,6 +215,7 @@ def fake_codex(self, *, events=None, result=None, exit_code=0, delay=0, read_std " ids=re.findall(r'\\b[0-9A-Fa-f]{8}(?:-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}\\b',text)\n" " audit.append({'event':'tool_call','id':call_id,'tool':item['tool']})\n" " audit.append({'event':'tool_result','id':call_id,'tool':item['tool'],'ok':item.get('status')=='completed' and not item.get('error') and not result.get('isError',False),'memory_ids':ids})\n" + f" if {audit_rows is not None!r}: audit={audit_rows!r}\n" " audit_path.write_text(''.join(json.dumps(item)+'\\n' for item in audit))\n" "result_path=pathlib.Path(args[args.index('--output-last-message')+1])\n" f"result_path.write_text(json.dumps({result!r}))\n" @@ -596,6 +597,48 @@ def test_provider_verified_write_completion_succeeds(self): self.assertEqual(result["write_calls"], 1) self.assertEqual(result["writes"][0]["memory_ids"], [MEMORY_ID]) + def no_write_rows(self): + return [{"event": "tool_call", "id": 1, "tool": "remember"}, + {"event": "tool_result", "id": 1, "tool": "remember", "ok": True, + "forwarded": True, "memory_ids": [], "write_outcome": "not_stored_near_duplicate"}] + + def test_provider_known_conflict_can_complete_with_no_new_memories(self): + result = self.provider_run(events=[{"type": "turn.completed"}], audit_rows=self.no_write_rows()) + self.assertEqual(result["status"], "succeeded") + self.assertEqual(result["write_calls"], 0) + self.assertEqual(result["tool_errors"], 0) + + def test_provider_known_conflict_cannot_support_stored_claim(self): + result = self.provider_run(events=[{"type": "turn.completed"}], audit_rows=self.no_write_rows(), + result={"outcome": "stored", "summary": "Only an existing ID was returned.", "memory_ids": [MEMORY_ID]}) + self.assertEqual(result["status"], "failed") + self.assertEqual(result["writes"], []) + + def test_provider_mixed_verified_write_and_conflict_counts_only_write(self): + rows = self.no_write_rows() + [ + {"event": "tool_call", "id": 2, "tool": "remember"}, + {"event": "tool_result", "id": 2, "tool": "remember", "ok": True, "memory_ids": [MEMORY_ID]}] + result = self.provider_run(events=[{"type": "turn.completed"}], audit_rows=rows, + result={"outcome": "stored", "summary": "One saved finding.", "memory_ids": [MEMORY_ID]}) + self.assertEqual(result["status"], "succeeded") + self.assertEqual(result["tool_calls"], 2) + self.assertEqual(result["write_calls"], 1) + self.assertEqual(result["writes"], [{"tool": "remember", "memory_ids": [MEMORY_ID]}]) + + def test_invalid_no_write_receipts_do_not_count_as_clean_completion(self): + mutations = [{"write_outcome": "unknown"}, {"memory_ids": [MEMORY_ID]}, + {"memory_ids": None}, {"forwarded": 1}, {"ok": 1}, + {"tool": "update"}, {"extra": "inconsistent"}] + for mutation in mutations: + with self.subTest(mutation=mutation): + rows = self.no_write_rows() + rows[-1].update(mutation) + path = self.base / "malformed-audit.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + observed = runner.audit_tools(path) + self.assertGreater(observed["tool_errors"], 0) + self.assertEqual(observed["writes"], []) + def test_claimed_memory_id_must_match_observed_successful_write(self): result = self.provider_run( events=[mcp_event(), mcp_event("remember", item_id="call-2"), {"type": "turn.completed"}], diff --git a/scripts/tests/test_codex_memory_proxy.py b/scripts/tests/test_codex_memory_proxy.py index 68a22cf..129515a 100644 --- a/scripts/tests/test_codex_memory_proxy.py +++ b/scripts/tests/test_codex_memory_proxy.py @@ -4,6 +4,7 @@ import json import os from pathlib import Path +import re import subprocess import sys import tempfile @@ -16,6 +17,20 @@ ID_A = "11111111-1111-1111-1111-111111111111" ID_B = "22222222-2222-2222-2222-222222222222" EDGE_ID = "33333333-3333-3333-3333-333333333333" +REPO_ROOT = MODULE_PATH.parents[2] +PLUGIN_SPEC = importlib.util.spec_from_file_location( + "packaged_memory_proxy", REPO_ROOT / "codex/plugins/engram/scripts/codex_learner/memory_proxy.py") +packaged_proxy = importlib.util.module_from_spec(PLUGIN_SPEC) +PLUGIN_SPEC.loader.exec_module(packaged_proxy) + +# The native contract is literal here so a production parser change cannot also +# silently change the fixture it is being tested against. +CONFLICT_PREFIX = "⚠️ Near-duplicate memory detected. The new memory was NOT stored.\n\nExisting similar memories:" +CONFLICT_SUFFIX = ('\n\nTo resolve:' + '\n - Use `update(id: "UUID", ...)` to modify the existing memory' + '\n - Use `remember(..., force: true)` to keep both' + '\n - Use `forget(id: "UUID")` to remove the old one, then `remember` the new one') +CONFLICT_TEXT = CONFLICT_PREFIX + f"\n [id:{ID_A}] (distance: 0.123, term overlap: 90%) PRIVATE_CONFLICT_SENTINEL" + CONFLICT_SUFFIX class CaptureAudit: @@ -129,6 +144,96 @@ def test_repeated_request_ids_cannot_overwrite_pending_audit(self): self.assertEqual(self.policy.pending[proxy.id_key(1)]["tool"], "recall") +class NoWriteContractTests(unittest.TestCase): + def policies(self, *, max_writes=3): + for module in (proxy, packaged_proxy): + audit = CaptureAudit() + yield module, audit, module.Policy(audit, "fixture", 8, max_writes) + + def test_native_warning_is_forwarded_with_only_vetted_no_write_metadata(self): + for module, audit, policy in self.policies(): + with self.subTest(module=module.__name__): + policy.client_message(call(1, "remember", {"content": "private"})) + native = result(1, CONFLICT_TEXT) + forwarded, child = policy.server_message(native) + self.assertEqual(forwarded, native) + self.assertIsNone(child) + self.assertEqual(audit.records[-1], {"event": "tool_result", "id": 1, + "tool": "remember", "ok": True, "forwarded": True, + "memory_ids": [], "write_outcome": "not_stored_near_duplicate"}) + self.assertTrue(module.verified_no_write_receipt(audit.records[-1])) + self.assertNotIn(ID_A, json.dumps(audit.records)) + self.assertNotIn("PRIVATE_CONFLICT_SENTINEL", json.dumps(audit.records)) + + def test_unknown_embedded_truncated_and_mixed_responses_stay_unverified(self): + cases = [result(1, "Unknown outcome " + ID_A), + result(1, "Quoted memory: " + CONFLICT_TEXT), + result(1, CONFLICT_PREFIX), result(1, CONFLICT_TEXT + "extra"), + result(1, CONFLICT_TEXT.replace("NOT stored", "stored"))] + mixed = result(1, CONFLICT_TEXT) + mixed["result"]["content"].append({"type": "text", "text": f"Stored memory (id: {ID_B})"}) + cases.append(mixed) + structured = result(1, CONFLICT_TEXT) + structured["result"]["structuredContent"] = {"stored": True} + cases.append(structured) + for native in cases: + for module, audit, policy in self.policies(): + with self.subTest(module=module.__name__, response=native): + policy.client_message(call(1, "remember", {"content": "private"})) + output, _ = policy.server_message(native) + self.assertTrue(output["result"]["isError"]) + self.assertIs(audit.records[-1]["ok"], False) + self.assertNotIn("write_outcome", audit.records[-1]) + + def test_non_boolean_error_flags_and_malformed_content_never_prove_no_write(self): + cases = [] + for flag in (True, None, 0, 1, "false", []): + cases.append(result(1, CONFLICT_TEXT, flag)) + missing = result(1, CONFLICT_TEXT) + del missing["result"]["isError"] + cases.append(missing) + for content in (None, {}, "bad", [None], [{"type": "text", "text": None}]): + native = result(1, CONFLICT_TEXT) + native["result"]["content"] = content + cases.append(native) + for native in cases: + for module, audit, policy in self.policies(): + with self.subTest(module=module.__name__, response=native): + policy.client_message(call(1, "remember", {"content": "private"})) + policy.server_message(native) + self.assertIs(audit.records[-1]["ok"], False) + self.assertNotIn("write_outcome", audit.records[-1]) + + def test_conflict_does_not_verify_other_tools_and_consumes_attempt_budget(self): + for module, audit, policy in self.policies(max_writes=1): + with self.subTest(module=module.__name__): + policy.client_message(call(1, "remember", {"content": "private"})) + policy.server_message(result(1, CONFLICT_TEXT)) + forwarded, denied = policy.client_message(call(2, "update", {"id": ID_A, "append": "new"})) + self.assertIsNone(forwarded) + self.assertTrue(denied["result"]["isError"]) + self.assertEqual(policy.write_calls, 2) + self.assertEqual(policy.total_calls, 2) + for module, audit, policy in self.policies(): + policy.client_message(call(1, "update", {"id": ID_A, "append": "new"})) + policy.server_message(result(1, CONFLICT_TEXT)) + self.assertIs(audit.records[-1]["ok"], False) + + def test_parser_contract_matches_native_source_and_packaged_copy(self): + native = (REPO_ROOT / "Sources/EngramKit/MemoryTools+Core.swift").read_text() + start = native.index('var warning = ') + end = native.index('return CallTool.Result(content: [.text(warning)], isError: false)', start) + body = native[start:end] + prefix = re.search(r'var warning = (".*")', body)[1] + suffix = ''.join(json.loads(value) for value in re.findall(r'warning \+= (".*")', body) + if '\\(mGid' not in value) + self.assertEqual(json.loads(prefix), CONFLICT_PREFIX) + self.assertEqual(suffix, CONFLICT_SUFFIX) + for module in (proxy, packaged_proxy): + self.assertEqual(module.NEAR_DUPLICATE_PREFIX, CONFLICT_PREFIX) + self.assertEqual(module.NEAR_DUPLICATE_SUFFIX, CONFLICT_SUFFIX) + + class TransportTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() From 33ca2839023ff5910f0dab3b9e0842f34d39115a Mon Sep 17 00:00:00 2001 From: jsflax Date: Mon, 21 Sep 2026 13:45:07 -0400 Subject: [PATCH 2/8] Preserve Codex learner identity across APFS device renumbering --- .../fixtures/identity_v1/SOURCE.json | 8 + .../fixtures/identity_v1/admission.py | 286 ++++++ .../fixtures/identity_v1/host_admission.py | 184 ++++ Tests/codex_plugin/run_suite.py | 1 + Tests/codex_plugin/test_bootstrap.py | 35 +- Tests/codex_plugin/test_entry_integration.py | 6 +- Tests/codex_plugin/test_file_identity.py | 375 ++++++++ Tests/codex_plugin/test_hook_status.py | 24 + Tests/codex_plugin/test_host_v2.py | 376 ++++++++ Tests/codex_plugin/test_migration.py | 670 ++++++++++++++ Tests/codex_plugin/test_v2_runtime.py | 175 ++++ codex/plugins/engram/README.md | 31 + .../engram/scripts/codex_learner/admission.py | 87 +- .../scripts/codex_learner/file_identity.py | 220 +++++ .../scripts/codex_learner/host_admission.py | 78 +- .../engram/scripts/codex_learner/migration.py | 856 ++++++++++++++++++ .../engram/scripts/codex_learner/runner.py | 75 +- .../scripts/codex_learner/runtime_identity.py | 3 + .../scripts/codex_learner/transcript.py | 56 +- .../engram/scripts/codex_learner_migrate.py | 10 + codex/plugins/engram/scripts/engram_hook.py | 25 +- .../plugins/engram/scripts/learner_router.py | 16 +- .../engram/skills/hook-status/SKILL.md | 2 + .../skills/hook-status/scripts/hook_status.py | 35 +- scripts/package_codex_plugin.py | 1 + 25 files changed, 3570 insertions(+), 65 deletions(-) create mode 100644 Tests/codex_plugin/fixtures/identity_v1/SOURCE.json create mode 100644 Tests/codex_plugin/fixtures/identity_v1/admission.py create mode 100644 Tests/codex_plugin/fixtures/identity_v1/host_admission.py create mode 100644 Tests/codex_plugin/test_file_identity.py create mode 100644 Tests/codex_plugin/test_host_v2.py create mode 100644 Tests/codex_plugin/test_migration.py create mode 100644 Tests/codex_plugin/test_v2_runtime.py create mode 100644 codex/plugins/engram/scripts/codex_learner/file_identity.py create mode 100644 codex/plugins/engram/scripts/codex_learner/migration.py create mode 100644 codex/plugins/engram/scripts/codex_learner_migrate.py diff --git a/Tests/codex_plugin/fixtures/identity_v1/SOURCE.json b/Tests/codex_plugin/fixtures/identity_v1/SOURCE.json new file mode 100644 index 0000000..524eb0e --- /dev/null +++ b/Tests/codex_plugin/fixtures/identity_v1/SOURCE.json @@ -0,0 +1,8 @@ +{ + "source_commit": "571b47196ae78accf36412aac82f2751c0f11c78", + "purpose": "Exact retained v1 policy parsers for migration suspension compatibility tests", + "files": { + "admission.py": "9ff002bb7644246cbe47cb5cbbf705c1276de5788bde99935fc1c50a4ab3d5f3", + "host_admission.py": "a327061097b355a75799977d1d824ab04fee041512c1371742b0fd96690f788f" + } +} diff --git a/Tests/codex_plugin/fixtures/identity_v1/admission.py b/Tests/codex_plugin/fixtures/identity_v1/admission.py new file mode 100644 index 0000000..b64efaf --- /dev/null +++ b/Tests/codex_plugin/fixtures/identity_v1/admission.py @@ -0,0 +1,286 @@ +"""Explicit GUI frontier admission; no enrollment occurs inside hooks. + +Local Codex origin metadata is trusted evidence, not authentication against +forgery by the same user. This module never enrolls sessions or writes state. +""" +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import uuid + +MAX_BYTES = 1024 * 1024 +LINEAGE_KEYS = ( + "forked_from_id", "parent_thread_id", "forked_from_ordinal_exclusive", + "subagent_history_start_ordinal", "history_base", +) + + +def _require(condition, reason): + if not condition: + raise ValueError("admission_" + reason) + + +def _object(pairs): + result = {} + for key, value in pairs: + _require(key not in result, "duplicate_json_key") + result[key] = value + return result + + +def _json(raw): + def invalid_constant(value): + raise ValueError("admission_nonfinite_json") + try: + value = json.loads(raw.decode("utf-8"), object_pairs_hook=_object, + parse_constant=invalid_constant) + except (UnicodeError, ValueError, RecursionError) as error: + raise ValueError("admission_invalid_json") from error + _require(isinstance(value, dict), "json_not_object") + return value + + +def _canonical(value): + _require(isinstance(value, str) and bool(value), "invalid_path") + path = Path(value) + _require(path.is_absolute() and str(path) == value + and path.resolve(strict=True) == path, "noncanonical_path") + return path + + +def _directory(path): + info = path.stat(follow_symlinks=False) + _require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid(), + "directory_not_owned") + return info + + +def _bound_directory(value): + _require(isinstance(value, dict) and set(value) == {"path", "device", "inode"}, + "invalid_directory_binding") + path = _canonical(value["path"]) + info = _directory(path) + _require(all(type(value[key]) is int and value[key] >= 0 for key in ("device", "inode")) + and (info.st_dev, info.st_ino) == (value["device"], value["inode"]), + "directory_identity_changed") + return path + + +def _owned_bytes(path, *, first_line=False, private=False): + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK + with os.fdopen(os.open(path, flags), "rb") as stream: + info = os.fstat(stream.fileno()) + _require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid(), + "file_not_owned_regular") + _require(not private or not info.st_mode & 0o022, "policy_writable_by_others") + _require(first_line or info.st_size <= MAX_BYTES, "record_too_large") + raw = stream.readline(MAX_BYTES + 1) if first_line else stream.read(MAX_BYTES + 1) + _require(len(raw) <= MAX_BYTES, "record_too_large") + _require(not first_line or raw.endswith(b"\n"), "initial_metadata_incomplete") + return raw, info + + +def _uuid(value, version): + _require(isinstance(value, str), "invalid_uuid") + try: + parsed = uuid.UUID(value) + except (ValueError, AttributeError) as error: + raise ValueError("admission_invalid_uuid") from error + _require(str(parsed) == value and parsed.variant == uuid.RFC_4122 + and parsed.version == version, "unqualified_uuid") + return parsed + + +def _timestamp(value): + _require(isinstance(value, str), "timestamp_missing") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + _require(parsed.tzinfo is not None and parsed.utcoffset() is not None, + "timestamp_without_timezone") + return parsed.astimezone(timezone.utc) + except (ValueError, OverflowError) as error: + raise ValueError("admission_invalid_timestamp") from error + + +def _utc(value): + return value.isoformat().replace("+00:00", "Z") + + +GUI_VERSION = "0.154.0-alpha.6.2" +# Exact task-scoped origin exception proposed for separate review, not a version allowlist. +# Initial metadata remains unchanged on resume and is sealed into enrollment. +ORIGIN_VERSION_BY_SESSION = { + "01a07cab-4062-79e1-99ca-14802ffd7142": "0.153.2", +} +ANCHOR_BYTES = 4096 +ORIGIN_FIELDS = {"session_id", "transcript_path", "device", "inode", "uid", + "initial_meta_sha256", "initial_meta_bytes", "source", "cli_version", + "origin_id_timestamp", "origin_metadata_timestamp"} +FRONTIER_FIELDS = {"frontier_offset", "frontier_anchor_start", "frontier_anchor_sha256"} + + +def _validated(action): + try: + return action() + except ValueError as error: + if str(error).startswith("admission_"): + raise + raise ValueError("admission_unavailable_or_invalid") from error + except (OSError, TypeError, KeyError, OverflowError, RuntimeError) as error: + raise ValueError("admission_unavailable_or_invalid") from error + + +def _origin(project, sessions, path, sid): + origin = _uuid(sid, 7) + path = _canonical(str(path)) + _require(path.is_relative_to(sessions), "transcript_outside_sessions") + parts = path.relative_to(sessions).parts + _require(len(parts) == 4 and re.fullmatch(r"\d{4}/\d{2}/\d{2}", "/".join(parts[:3])), + "invalid_rollout_layout") + match = re.fullmatch(r"rollout-(" + re.escape("-".join(parts[:3])) + + r"T\d{2}-\d{2}-\d{2})-" + re.escape(sid) + r"\.jsonl", parts[3]) + _require(match is not None, "invalid_rollout_filename") + try: + datetime.strptime(match.group(1), "%Y-%m-%dT%H-%M-%S") + except ValueError as error: + raise ValueError("admission_invalid_rollout_date") from error + raw, info = _owned_bytes(path, first_line=True) + record = _json(raw) + payload = record.get("payload") + _require(record.get("type") == "session_meta" and isinstance(payload, dict), "initial_metadata_required") + _require(payload.get("id") == sid and ("session_id" not in payload or payload["session_id"] == sid), + "metadata_session_mismatch") + _require(payload.get("cwd") == str(project), "metadata_project_mismatch") + required_version = ORIGIN_VERSION_BY_SESSION.get(sid, GUI_VERSION) + _require(payload.get("source") == "vscode" and payload.get("cli_version") == required_version + and payload.get("history_mode") == "paginated", "unqualified_gui_metadata") + _require(all(payload.get(key) is None for key in LINEAGE_KEYS), "inherited_session") + created = _timestamp(payload.get("timestamp")) + origin_time = datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(milliseconds=origin.int >> 80) + _require(origin_time <= created <= datetime.now(timezone.utc), "invalid_origin_time") + return {"session_id": sid, "transcript_path": str(path), "device": info.st_dev, + "inode": info.st_ino, "uid": info.st_uid, "initial_meta_bytes": len(raw), + "initial_meta_sha256": hashlib.sha256(raw).hexdigest(), "source": payload["source"], + "cli_version": payload["cli_version"], "origin_id_timestamp": _utc(origin_time), + "origin_metadata_timestamp": _utc(created)} + + +def _frontier(origin, offset=None): + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK + with os.fdopen(os.open(origin["transcript_path"], flags), "rb") as stream: + info = os.fstat(stream.fileno()) + _require(stat.S_ISREG(info.st_mode) and (info.st_dev, info.st_ino, info.st_uid) + == (origin["device"], origin["inode"], os.getuid()), "transcript_identity_changed") + # Capture only the observed EOF. A partial tail is never rounded backwards. + offset = info.st_size if offset is None else offset + _require(type(offset) is int and origin["initial_meta_bytes"] <= offset <= info.st_size, + "frontier_truncated_or_invalid") + start = max(0, offset - ANCHOR_BYTES) + stream.seek(start) + data = stream.read(offset - start) + _require(len(data) == offset - start and data.endswith(b"\n"), "frontier_not_complete_line") + return {"frontier_offset": offset, "frontier_anchor_start": start, + "frontier_anchor_sha256": hashlib.sha256(data).hexdigest()} + + +def capture(project: dict, sessions_dir: dict, transcript_path: str, session_id: str) -> dict: + """Read-only, explicitly invoked capture; installer owns publication and activation. + + Reads initial metadata and at most 4096 boundary bytes. Returns hashes only, + never history text. This function is never called by check or the hook path. + """ + def run(): + origin = _origin(_bound_directory(project), _bound_directory(sessions_dir), transcript_path, session_id) + frontier = _frontier(origin) + return {**origin, **frontier, "captured_at": _utc(datetime.now(timezone.utc))} + return _validated(run) + + +def _policy(root): + root = _canonical(str(root)) + _directory(root) + raw, _ = _owned_bytes(root / "admission.json", private=True) + policy = _json(raw) + _require(policy.get("enabled") is True, "inactive") + _require(set(policy) == {"schema_version", "mode", "enabled", "activation_id", "cutoff", + "state_dir", "project", "sessions_dir", "enrollments"}, "invalid_policy_schema") + _require(type(policy["schema_version"]) is int and policy["schema_version"] == 1 + and policy["mode"] == "explicit_frontier_v1", "unsupported_policy") + _uuid(policy["activation_id"], 4) + cutoff = _timestamp(policy["cutoff"]) + _require(policy["state_dir"] == str(root), "state_dir_mismatch") + project, sessions = _bound_directory(policy["project"]), _bound_directory(policy["sessions_dir"]) + enrolled = policy["enrollments"] + _require(isinstance(enrolled, dict) and len(enrolled) == 1, "one_explicit_enrollment_required") + sid, entry = next(iter(enrolled.items())) + _require(isinstance(entry, dict) and set(entry) == ORIGIN_FIELDS | FRONTIER_FIELDS | {"captured_at"}, + "invalid_enrollment_schema") + _require(all(type(entry[key]) is int and entry[key] >= 0 for key in + ("device", "inode", "uid", "initial_meta_bytes", "frontier_offset", "frontier_anchor_start")), + "invalid_enrollment_numbers") + actual = _origin(project, sessions, entry["transcript_path"], sid) + _require(all(entry[key] == actual[key] for key in ORIGIN_FIELDS), "enrollment_origin_changed") + captured = _timestamp(entry["captured_at"]) + _require(cutoff <= captured <= datetime.now(timezone.utc) + and _timestamp(actual["origin_metadata_timestamp"]) <= captured, "invalid_capture_time") + frontier = _frontier(actual, entry["frontier_offset"]) + _require(all(entry[key] == frontier[key] for key in FRONTIER_FIELDS), "frontier_anchor_changed") + return policy, raw, entry + + +def check_activation(root: Path) -> dict: + from . import host_admission + if host_admission.mode(root): + return _validated(lambda: host_admission.policy(root)[0]) + return _validated(lambda: _policy(root)[0]) + + +def enrollment_ids(root: Path, policy: dict) -> list[str]: + if policy.get("mode") == "host_sessions_v1": + from .host_admission import enrollment_ids as host_ids + return host_ids(root) + return list(policy["enrollments"]) + + +def check(root: Path, request: dict) -> dict: + from . import host_admission + if host_admission.mode(root): + return _validated(lambda: host_admission.check(root, request)) + def run(): + _require(isinstance(request, dict) and request.get("event") == "Stop", "unqualified_event") + _require(isinstance(request.get("turn_id"), str) + and re.fullmatch(r"[A-Za-z0-9_-]{1,100}", request["turn_id"]) is not None, + "missing_or_invalid_turn") + policy, raw, entry = _policy(root) + sid = request.get("session_id") + _require(sid == entry["session_id"] and request.get("hook_session_id") == sid, "session_not_enrolled") + _require(request.get("cwd") == policy["project"]["path"] + and request.get("hook_cwd") == policy["project"]["path"], "project_mismatch") + _require(all(request.get(key) == entry[key] for key in ("transcript_path", "device", "inode")) + and all(type(request.get(key)) is int for key in ("device", "inode")), "transcript_identity_changed") + return {**entry, "activation_id": policy["activation_id"], "cutoff": policy["cutoff"], + "policy_sha256": hashlib.sha256(raw).hexdigest(), "project": dict(policy["project"]), + "state_dir": policy["state_dir"], "mode": policy["mode"]} + return _validated(run) + + +def state_paths(root: Path, sid: str) -> dict: + """Reject symlink/special-file state paths before cursor/receipt adoption.""" + _uuid(sid, 7) + result = {} + for name in ("admissions", "sessions", "pending"): + parent = root / name + _require(not parent.is_symlink(), "state_directory_symlink") + if parent.exists(): + _directory(parent) + path = parent / (sid + ".json") + _require(not path.is_symlink(), "state_file_symlink") + if path.exists(): + info = path.stat(follow_symlinks=False) + _require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid(), "state_not_owned_regular") + result[name] = path + return result diff --git a/Tests/codex_plugin/fixtures/identity_v1/host_admission.py b/Tests/codex_plugin/fixtures/identity_v1/host_admission.py new file mode 100644 index 0000000..e9c920b --- /dev/null +++ b/Tests/codex_plugin/fixtures/identity_v1/host_admission.py @@ -0,0 +1,184 @@ +"""Automatic per-task frontiers for one explicitly enabled local Codex host. + +The user's installation enables this host policy, not a task allowlist. A task's +first observed event records EOF and never backfills existing history. Subsequent +Stop and automatic PreCompact events can learn only appended visible messages. +Local metadata is provenance, not authentication against the account owner. +""" +from datetime import datetime, timezone +import hashlib +import os +from pathlib import Path +import re + +from . import admission +from .transcript import inspect_rollout + +MODE = "host_sessions_v1" +OBSERVE_EVENTS = {"SessionStart", "UserPromptSubmit", "SubagentStart", "SubagentStop", "Stop", "PreCompact", "SessionEnd"} +LEARN_EVENTS = {"Stop", "SubagentStop", "PreCompact"} + + +def mode(root): + raw, _ = admission._owned_bytes(root / "admission.json", private=True) + return admission._json(raw).get("mode") == MODE + + +def policy(root): + root = admission._canonical(str(root)) + info = admission._directory(root) + admission._require(not info.st_mode & 0o077, "state_directory_not_private") + raw, _ = admission._owned_bytes(root / "admission.json", private=True) + value = admission._json(raw) + admission._require(set(value) == {"schema_version", "mode", "enabled", "activation_id", "cutoff", + "state_dir", "sessions_dir"}, "invalid_host_policy_schema") + admission._require(type(value["schema_version"]) is int and value["schema_version"] == 1 + and value["mode"] == MODE, "unsupported_policy") + admission._require(value["enabled"] is True, "inactive") + admission._uuid(value["activation_id"], 4) + admission._require(admission._timestamp(value["cutoff"]) <= datetime.now(timezone.utc), "future_cutoff") + admission._require(value["state_dir"] == str(root), "state_dir_mismatch") + admission._bound_directory(value["sessions_dir"]) + return value, raw + + +def _origin(value, payload): + sid = payload.get("session_id") + path = admission._canonical(payload.get("transcript_path")) + sessions = admission._bound_directory(value["sessions_dir"]) + admission._require(path.is_relative_to(sessions), "transcript_outside_sessions") + parts = path.relative_to(sessions).parts + admission._require(len(parts) == 4 and re.fullmatch(r"\d{4}/\d{2}/\d{2}", "/".join(parts[:3])), + "invalid_rollout_layout") + raw, info = admission._owned_bytes(path, first_line=True) + record = admission._json(raw) + meta = record.get("payload") + admission._require(record.get("type") == "session_meta" and isinstance(meta, dict), "initial_metadata_required") + canonical_sid = meta.get("id", meta.get("session_id")) + admission._uuid(canonical_sid, 7) + admission._require(re.fullmatch(r"rollout-" + re.escape("-".join(parts[:3])) + + r"T\d{2}-\d{2}-\d{2}-" + re.escape(canonical_sid) + r"\.jsonl", parts[3]) is not None, + "invalid_rollout_filename") + parsed = inspect_rollout(path) + admission._require(parsed.session_id == canonical_sid and parsed.fork_boundary_known, "unknown_inherited_boundary") + if sid != canonical_sid: + source = parsed.source if isinstance(parsed.source, dict) else {} + subagent = source.get("subagent") + spawn = subagent.get("thread_spawn") if isinstance(subagent, dict) else None + admission._require(isinstance(spawn, dict) and bool(spawn.get("parent_thread_id")) + and sid in {parsed.parent_session_id, parsed.hook_session_id}, "metadata_session_mismatch") + admission._require(payload.get("agent_id") in (None, canonical_sid), "metadata_agent_mismatch") + source = meta.get("source") + admission._require(isinstance(source, (str, dict)) and bool(source), "unqualified_source") + admission._require("engram-session-learner" not in str(source), "learner_transcript") + version = meta.get("cli_version") + admission._require(isinstance(version, str) and 0 < len(version) <= 128, "unqualified_version") + project = admission._canonical(meta.get("cwd")) + project_info = admission._directory(project) + admission._require(payload.get("cwd") == str(project), "project_mismatch") + created = admission._timestamp(meta.get("timestamp")) + admission._require(created <= datetime.now(timezone.utc), "invalid_origin_time") + return {"session_id": canonical_sid, "transcript_path": str(path), "device": info.st_dev, + "inode": info.st_ino, "uid": info.st_uid, "initial_meta_bytes": len(raw), + "initial_meta_sha256": hashlib.sha256(raw).hexdigest(), "source": source, + "cli_version": version, "origin_metadata_timestamp": admission._utc(created), + "project": {"path": str(project), "device": project_info.st_dev, "inode": project_info.st_ino}} + + +def _entry_path(root, sid): + admission._uuid(sid, 7) + parent = root / "enrollments" + admission._require(not parent.is_symlink(), "state_directory_symlink") + if parent.exists(): + admission._directory(parent) + path = parent / (sid + ".json") + admission._require(not path.is_symlink(), "state_file_symlink") + return path + + +def _validate_entry(root, value, payload): + origin = _origin(value, payload) + raw, _ = admission._owned_bytes(_entry_path(root, origin["session_id"]), private=True) + entry = admission._json(raw) + admission._require(set(entry) == set(origin) | admission.FRONTIER_FIELDS | {"captured_at", "activation_id"}, + "invalid_enrollment_schema") + admission._require(entry["activation_id"] == value["activation_id"], "enrollment_activation_changed") + admission._require(all(entry[key] == item for key, item in origin.items()), "enrollment_origin_changed") + captured = admission._timestamp(entry["captured_at"]) + admission._require(admission._timestamp(value["cutoff"]) <= captured <= datetime.now(timezone.utc), + "invalid_capture_time") + frontier = admission._frontier(origin, entry["frontier_offset"]) + admission._require(all(entry[key] == item for key, item in frontier.items()), "frontier_anchor_changed") + return entry + + +def observe(root, payload): + """Register a task at observed EOF; caller holds the root's enqueue lock. + + Returns (canonical task id, newly enrolled, fresh child eligible immediately). Existing entries are validated, + never re-captured, even on SessionStart/resume/compaction or package upgrades. + """ + value, _ = policy(root) + event = payload.get("hook_event_name") + admission._require(event in OBSERVE_EVENTS, "unqualified_event") + if event == "PreCompact": + admission._require(payload.get("trigger") == "auto", "manual_compaction_not_enabled") + origin = _origin(value, payload) + path = _entry_path(root, origin["session_id"]) + if path.exists(): + _validate_entry(root, value, payload) + return origin["session_id"], False, False + # Never adopt state from another admission or silently reset missing entries. + for state_path in admission.state_paths(root, origin["session_id"]).values(): + admission._require(not state_path.exists(), "unowned_existing_state") + parsed = inspect_rollout(origin["transcript_path"]) + # A newly spawned child can finish before any child-start hook exposes its + # transcript. Its known inherited-history boundary keeps the initial visible + # excerpt child-only, and the cutoff excludes pre-installation children. + source = parsed.source if isinstance(parsed.source, dict) else {} + subagent = source.get("subagent") + spawn = subagent.get("thread_spawn") if isinstance(subagent, dict) else None + fresh_child = (event == "SubagentStop" and isinstance(spawn, dict) + and bool(spawn.get("parent_thread_id")) and parsed.fork_boundary_known + and admission._timestamp(origin["origin_metadata_timestamp"]) >= admission._timestamp(value["cutoff"])) + frontier = admission._frontier(origin, parsed.start_offset if fresh_child else None) + entry = {**origin, **frontier, "captured_at": admission._utc(datetime.now(timezone.utc)), + "activation_id": value["activation_id"]} + # Import here to avoid a module cycle; publication is a metadata-only atomic write. + from .runner import atomic_json + atomic_json(path, entry) + return origin["session_id"], True, fresh_child + + +def check(root, request): + value, raw = policy(root) + admission._require(request.get("event") in LEARN_EVENTS, "unqualified_event") + if request["event"] == "Stop": + admission._require(isinstance(request.get("turn_id"), str) + and re.fullmatch(r"[A-Za-z0-9_-]{1,100}", request["turn_id"]), "missing_or_invalid_turn") + elif request["event"] == "PreCompact": + admission._require(request.get("trigger") == "auto", "manual_compaction_not_enabled") + payload = {"session_id": request.get("hook_session_id"), "transcript_path": request.get("transcript_path"), + "cwd": request.get("hook_cwd"), "agent_id": request.get("session_id")} + entry = _validate_entry(root, value, payload) + admission._require(request.get("session_id") == entry["session_id"] + and request.get("cwd") == entry["project"]["path"], "session_or_project_mismatch") + admission._require(all(request.get(key) == entry[key] for key in ("device", "inode")), "transcript_identity_changed") + return {**entry, "cutoff": value["cutoff"], "policy_sha256": hashlib.sha256(raw).hexdigest(), + "state_dir": str(root), "mode": MODE} + + +def enrollment_ids(root): + parent = root / "enrollments" + admission._require(not parent.is_symlink(), "state_directory_symlink") + if not parent.exists(): + return [] + admission._directory(parent) + result = [] + for path in parent.glob("*.json"): + try: + admission._uuid(path.stem, 7) + except ValueError: + continue + result.append(path.stem) + return sorted(result) diff --git a/Tests/codex_plugin/run_suite.py b/Tests/codex_plugin/run_suite.py index 16d018b..ce7a93f 100644 --- a/Tests/codex_plugin/run_suite.py +++ b/Tests/codex_plugin/run_suite.py @@ -18,6 +18,7 @@ "test_budget_guidance", "test_lock_inheritance", "test_hook_parity", "test_hook_status", "test_bootstrap", "test_entry_integration", "test_memory_policy", + "test_file_identity", "test_host_v2", "test_v2_runtime", "test_migration", ) def run_module(name): diff --git a/Tests/codex_plugin/test_bootstrap.py b/Tests/codex_plugin/test_bootstrap.py index aa22d7e..c125bba 100644 --- a/Tests/codex_plugin/test_bootstrap.py +++ b/Tests/codex_plugin/test_bootstrap.py @@ -4,10 +4,14 @@ from pathlib import Path import tempfile import unittest +import sys +from unittest import mock from suite_support import PLUGIN_ROOT, REPOSITORY_ROOT, TEMP_ROOT HERE = REPOSITORY_ROOT +sys.path.insert(0, str(PLUGIN_ROOT / "scripts")) +from codex_learner import file_identity spec = importlib.util.spec_from_file_location("bootstrap", PLUGIN_ROOT / "scripts/engram_hook.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -19,6 +23,10 @@ def setUp(self): self.addCleanup(self.temp.cleanup) self.home = Path(self.temp.name) (self.home / "sessions").mkdir() + capture = mock.patch.object(file_identity, "capture_fd", side_effect=lambda fd: { + "scheme": "macos_volume_uuid_inode_v1", + "volume_uuid": "11111111-2222-4333-8444-555555555555", "inode": os.fstat(fd).st_ino}) + capture.start(); self.addCleanup(capture.stop) def test_portable_state_and_no_pilot_pins(self): root = module.initialize(self.home) @@ -27,7 +35,9 @@ def test_portable_state_and_no_pilot_pins(self): self.assertEqual(recall["selected_memory_ids"], []) self.assertTrue(recall["semantic_recall"]) self.assertEqual(policy["sessions_dir"]["path"], str(self.home / "sessions")) - self.assertEqual(policy["mode"], "host_sessions_v1") + self.assertEqual(policy["mode"], "host_sessions_v2") + self.assertEqual(policy["schema_version"], 2) + self.assertEqual(set(policy["sessions_dir"]), {"path", "identity"}) self.assertEqual(root.stat().st_mode & 0o777, 0o700) for path in root.rglob("*.json"): self.assertEqual(path.stat().st_mode & 0o777, 0o600) @@ -44,6 +54,29 @@ def test_disabled_policy_and_custom_settings_survive(self): module.initialize(self.home) self.assertEqual(before, {p: p.read_bytes() for p in root.rglob("*.json")}) + def test_existing_v1_policy_is_not_seeded_with_current_volume(self): + root = module.initialize(self.home) + path = root / "learner/admission.json" + value = json.loads(path.read_bytes()) + info = (self.home / "sessions").stat() + value.update(schema_version=1, mode="host_sessions_v1", + sessions_dir={"path": str(self.home / "sessions"), "device": info.st_dev, "inode": info.st_ino}) + path.write_text(json.dumps(value)) + route = root / "learner-routes.json" + route.write_text(json.dumps({"schema_version": 1, "mode": "host_sessions_v1", + "enabled": False, "state_dir": str(root / "learner")})) + before = {p: p.read_bytes() for p in root.rglob("*.json")} + with mock.patch.object(file_identity, "capture_fd", side_effect=AssertionError("implicit migration")): + module.initialize(self.home) + self.assertEqual(before, {p: p.read_bytes() for p in root.rglob("*.json")}) + + def test_unavailable_stable_identity_never_creates_legacy_fallback(self): + with mock.patch.object(file_identity, "capture_fd", side_effect=ValueError("admission_identity_unavailable")): + with self.assertRaises(ValueError): + module.initialize(self.home) + self.assertFalse((self.home / "engram/learner/admission.json").exists()) + self.assertFalse((self.home / "engram/learner-routes.json").exists()) + def test_symlink_root_rejected(self): other = self.home / "other" other.mkdir() diff --git a/Tests/codex_plugin/test_entry_integration.py b/Tests/codex_plugin/test_entry_integration.py index 9af7eaa..a3cd5df 100644 --- a/Tests/codex_plugin/test_entry_integration.py +++ b/Tests/codex_plugin/test_entry_integration.py @@ -14,11 +14,15 @@ HERE = REPOSITORY_ROOT sys.path.insert(0, str(PLUGIN_ROOT / "scripts")) import engram_hook -from codex_learner import runner +from codex_learner import runner, file_identity class EntryIntegration(unittest.TestCase): def setUp(self): + capture = mock.patch.object(file_identity, "capture_fd", side_effect=lambda fd: { + "scheme": "macos_volume_uuid_inode_v1", + "volume_uuid": "11111111-2222-4333-8444-555555555555", "inode": os.fstat(fd).st_ino}) + capture.start(); self.addCleanup(capture.stop) self.temp = tempfile.TemporaryDirectory(dir=TEMP_ROOT) self.addCleanup(self.temp.cleanup) self.home = Path(self.temp.name) diff --git a/Tests/codex_plugin/test_file_identity.py b/Tests/codex_plugin/test_file_identity.py new file mode 100644 index 0000000..9ebeac7 --- /dev/null +++ b/Tests/codex_plugin/test_file_identity.py @@ -0,0 +1,375 @@ +"""Persistent identity ABI and race tests; only owned files and fake syscalls. + +No native library, provider, database, subprocess or network is used. A separate +read-only qualification receipt binds actual Darwin behavior to a source hash. +""" +import ctypes +import dataclasses +import importlib.util +import os +from pathlib import Path +import struct +import sys +import tempfile +import unittest +from unittest import mock +import uuid + +from suite_support import PLUGIN_ROOT, TEMP_ROOT + + +SOURCE = PLUGIN_ROOT / "scripts/codex_learner/file_identity.py" +SPEC = importlib.util.spec_from_file_location("_file_identity_under_test", SOURCE) +IDENTITY = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = IDENTITY +SPEC.loader.exec_module(IDENTITY) +UUID = "e535dd8a-5390-470f-b07b-bb0c5dd6414b" +OTHER_UUID = "d535dd8a-5390-470f-b07b-bb0c5dd6414b" + + +def attributes(*, length=72, common=0x80000000, volume=0x60000, + capabilities=0x20001, valid=0x20001, volume_uuid=UUID): + return struct.pack("=14I16s", length, common, volume, 0, 0, 0, + capabilities, 0, 0, 0, valid, 0, 0, 0, + uuid.UUID(volume_uuid).bytes) + + +class ValidationTests(unittest.TestCase): + def test_returns_same_validated_dict_and_supports_full_inode_width(self): + value = {"scheme": IDENTITY.SCHEME, "volume_uuid": UUID, + "inode": (1 << 64) - 1} + self.assertIs(IDENTITY.validate(value), value) + + def test_rejects_nonexact_shape_and_scheme(self): + valid = {"scheme": IDENTITY.SCHEME, "volume_uuid": UUID, "inode": 12} + for value in [None, [], {}, {**valid, "device": 42}, + {**valid, "scheme": "device_inode_v1"}]: + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "admission_identity_"): + IDENTITY.validate(value) + + def test_rejects_noncanonical_and_zero_uuids(self): + for value in [None, 4, UUID.upper(), UUID.replace("-", ""), "{" + UUID + "}", + "0" * 36, "00000000-0000-0000-0000-000000000000"]: + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "uuid_invalid"): + IDENTITY.validate({"scheme": IDENTITY.SCHEME, "volume_uuid": value, "inode": 12}) + + def test_rejects_noninteger_and_out_of_range_inodes(self): + for value in [False, True, 0, -1, 1 << 64, 3.0, "3", None]: + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "inode_invalid"): + IDENTITY.validate({"scheme": IDENTITY.SCHEME, "volume_uuid": UUID, "inode": value}) + + +class AttributeTests(unittest.TestCase): + def test_accepts_fixed_sdk_layout(self): + self.assertEqual(IDENTITY._parse_volume(attributes()), UUID) + + def test_rejects_truncation_and_reported_overflow(self): + raw = attributes() + for value in [raw[:-1], raw + b"\0", attributes(length=71), attributes(length=73)]: + with self.subTest(length=len(value)), self.assertRaisesRegex(ValueError, "attributes_malformed"): + IDENTITY._parse_volume(value) + + def test_rejects_missing_or_unrequested_returned_masks(self): + for options in [{"common": 0}, {"common": 0x80000001}, {"volume": 0x20000}, + {"volume": 0x40000}, {"volume": 0x80060000}, {"volume": 0x60001}]: + with self.subTest(options=options), self.assertRaisesRegex(ValueError, "attributes_unsupported"): + IDENTITY._parse_volume(attributes(**options)) + + def test_rejects_invalid_or_missing_persistence_capabilities(self): + for field in ("capabilities", "valid"): + for value in (0, 1, 0x20000): + with self.subTest(field=field, value=value), self.assertRaisesRegex(ValueError, "persistence_unsupported"): + IDENTITY._parse_volume(attributes(**{field: value})) + + def test_accepts_additional_declared_capabilities(self): + self.assertEqual(IDENTITY._parse_volume(attributes(capabilities=0x20003, + valid=0xFFFFFFFF)), UUID) + + def test_rejects_zero_uuid_even_when_returned_mask_claims_support(self): + with self.assertRaisesRegex(ValueError, "uuid_invalid"): + IDENTITY._parse_volume(attributes(volume_uuid="00000000-0000-0000-0000-000000000000")) + + +class Function: + def __init__(self, call=lambda *args: 0): + self.call = call + self.argtypes = self.restype = None + + def __call__(self, *args): + return self.call(*args) + + +class NativeBoundaryTests(unittest.TestCase): + def test_sdk_layout_and_architecture_specific_symbol(self): + self.assertEqual(ctypes.sizeof(IDENTITY._StatFS64), 2168) + self.assertEqual(IDENTITY._StatFS64.f_fsid.offset, 48) + self.assertEqual(IDENTITY._StatFS64.f_mntonname.offset, 88) + self.assertEqual(ctypes.sizeof(IDENTITY._AttrList), 24) + for architecture, symbol in [("arm64", "fstatfs"), ("x86_64", "fstatfs$INODE64")]: + library = type("Library", (), {})() + function = Function() + setattr(library, symbol, function) + library.fgetattrlist = Function() + with self.subTest(architecture=architecture), mock.patch.object(IDENTITY.sys, "platform", "darwin"), \ + mock.patch.object(IDENTITY.platform, "machine", return_value=architecture), \ + mock.patch.object(IDENTITY.ctypes, "CDLL", return_value=library): + native = IDENTITY._Darwin() + self.assertIs(native.statfs, function) + self.assertIs(native.getattr.argtypes[-1], ctypes.c_uint) + + def test_unsupported_platform_never_loads_native_library(self): + with mock.patch.object(IDENTITY.sys, "platform", "linux"), \ + mock.patch.object(IDENTITY.ctypes, "CDLL") as load: + with self.assertRaisesRegex(ValueError, "platform_unsupported"): + IDENTITY._Darwin() + load.assert_not_called() + + def test_missing_native_symbol_is_bounded_failure(self): + with mock.patch.object(IDENTITY.sys, "platform", "darwin"), \ + mock.patch.object(IDENTITY.platform, "machine", return_value="arm64"), \ + mock.patch.object(IDENTITY.ctypes, "CDLL", return_value=object()): + with self.assertRaisesRegex(ValueError, "api_unavailable"): + IDENTITY._Darwin() + + def test_getattr_request_and_fixed_buffer(self): + native = IDENTITY._Darwin.__new__(IDENTITY._Darwin) + calls = [] + + def getattr_call(fd, pointer, buffer, size, options): + request = ctypes.cast(pointer, ctypes.POINTER(IDENTITY._AttrList)).contents + calls.append((fd, request.bitmapcount, request.reserved, request.commonattr, + request.volattr, request.dirattr, request.fileattr, + request.forkattr, size, options)) + ctypes.memmove(buffer, attributes(), size) + return 0 + + native.getattr = getattr_call + self.assertEqual(native.volume_uuid(17), UUID) + self.assertEqual(calls, [(17, 5, 0, 0x80000000, 0x80060000, 0, 0, 0, 72, 12)]) + + def test_getattr_syscall_failure_is_bounded(self): + native = IDENTITY._Darwin.__new__(IDENTITY._Darwin) + native.getattr = lambda *args: -1 + with self.assertRaisesRegex(ValueError, "attributes_unavailable"): + native.volume_uuid(17) + + def native_filesystem(self, *, kind=b"apfs", flags=0x1000, mount=b"/Volumes/test", result=0): + native = IDENTITY._Darwin.__new__(IDENTITY._Darwin) + + def statfs_call(fd, pointer): + value = ctypes.cast(pointer, ctypes.POINTER(IDENTITY._StatFS64)).contents + value.f_fsid[:] = (42, 26) + value.f_type = 26 + value.f_flags = flags + value.f_fstypename = kind + value.f_mntonname = mount + value.f_mntfromname = b"/dev/test" + return result + + native.statfs = statfs_call + return native + + def test_statfs_parsing_and_local_apfs_requirement(self): + fs = self.native_filesystem().filesystem(17) + self.assertEqual(fs.fsid, (42, 26)) + self.assertEqual(fs.mount, "/Volumes/test") + for options in [{"kind": b"hfs"}, {"kind": b"nfs"}, {"flags": 0}]: + with self.subTest(options=options), self.assertRaisesRegex(ValueError, "filesystem_unsupported"): + self.native_filesystem(**options).filesystem(17) + + def test_statfs_malformed_mount_and_unterminated_string(self): + for mount in [b"", b"relative", b"/Volumes/../test", b"x" * 1024]: + with self.subTest(mount=mount[:20]), self.assertRaisesRegex(ValueError, "(mount_invalid|filesystem_malformed)"): + self.native_filesystem(mount=mount).filesystem(17) + + def test_statfs_syscall_failure_is_bounded(self): + with self.assertRaisesRegex(ValueError, "filesystem_unavailable"): + self.native_filesystem(result=-1).filesystem(17) + + +class CaptureTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(dir=TEMP_ROOT, prefix="identity-") + self.addCleanup(self.temp.cleanup) + self.parent = Path(self.temp.name) + self.root = self.parent / "mount" + self.root.mkdir() + self.file = self.root / "target" + self.file.write_bytes(b"owned fixture\n") + self.fd = os.open(self.file, os.O_RDONLY) + self.addCleanup(os.close, self.fd) + self.fs = IDENTITY._Filesystem((42, 26), 26, 1, 0x1000, b"apfs", str(self.root), b"/dev/test") + self.backend = mock.Mock() + self.backend.filesystem.return_value = self.fs + self.backend.volume_uuid.return_value = UUID + patch = mock.patch.object(IDENTITY, "_Darwin", return_value=self.backend) + patch.start() + self.addCleanup(patch.stop) + + def assert_owned_fds_closed(self): + fds = {args[0] for args, _ in self.backend.filesystem.call_args_list} + for fd in fds: + self.assertNotEqual(fd, self.fd) + with self.assertRaises(OSError): + os.fstat(fd) + os.fstat(self.fd) + + def test_capture_retains_caller_and_uses_volume_root_without_seeking(self): + os.lseek(self.fd, 2, os.SEEK_SET) + result = IDENTITY.capture_fd(self.fd) + self.assertEqual(result, {"scheme": IDENTITY.SCHEME, "volume_uuid": UUID, + "inode": self.file.stat().st_ino}) + self.assertEqual(os.lseek(self.fd, 0, os.SEEK_CUR), 2) + root_fds = [args[0] for args, _ in self.backend.volume_uuid.call_args_list] + self.assertEqual(len(root_fds), 2) + self.assertEqual(root_fds[0], root_fds[1]) + self.assertEqual(self.backend.filesystem.call_count, 4) + self.assert_owned_fds_closed() + + def test_directory_target_supported(self): + fd = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY) + try: + self.assertEqual(IDENTITY.capture_fd(fd)["inode"], self.root.stat().st_ino) + finally: + os.close(fd) + + def test_invalid_fd_values_and_closed_fd(self): + for fd in [True, None, -1, 1 << 31]: + with self.subTest(fd=fd), self.assertRaisesRegex(ValueError, "fd_invalid"): + IDENTITY.capture_fd(fd) + closed = os.dup(self.fd) + os.close(closed) + with self.assertRaisesRegex(ValueError, "identity_unavailable"): + IDENTITY.capture_fd(closed) + + def test_fifo_target_rejected(self): + fifo = self.root / "fifo" + os.mkfifo(fifo) + fd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK) + try: + with self.assertRaisesRegex(ValueError, "file_type_unsupported"): + IDENTITY.capture_fd(fd) + finally: + os.close(fd) + + def test_root_filesystem_mismatch(self): + self.backend.filesystem.side_effect = [self.fs, dataclasses.replace(self.fs, fsid=(43, 26))] + with self.assertRaisesRegex(ValueError, "root_mismatch"): + IDENTITY.capture_fd(self.fd) + self.backend.volume_uuid.assert_not_called() + self.assert_owned_fds_closed() + + def test_root_current_device_mismatch(self): + real_fstat = os.fstat + root_inode = self.root.stat().st_ino + + def changed_device(fd): + result = real_fstat(fd) + if result.st_ino == root_inode: + fields = list(result) + fields[2] += 1 + return os.stat_result(fields) + return result + + with mock.patch.object(IDENTITY.os, "fstat", side_effect=changed_device): + with self.assertRaisesRegex(ValueError, "root_mismatch"): + IDENTITY.capture_fd(self.fd) + self.backend.volume_uuid.assert_not_called() + self.assert_owned_fds_closed() + + def test_mount_symlink_rejected_before_query(self): + alias = self.parent / "alias" + alias.symlink_to(self.root) + self.backend.filesystem.return_value = dataclasses.replace(self.fs, mount=str(alias)) + with self.assertRaisesRegex(ValueError, "mount_invalid"): + IDENTITY.capture_fd(self.fd) + self.backend.volume_uuid.assert_not_called() + self.assert_owned_fds_closed() + + def test_root_path_replacement_detected(self): + def replace_root(fd): + if self.root.exists(): + self.root.rename(self.parent / "moved") + self.root.mkdir() + return UUID + + # Replace only once; the second volume query must still return UUID. + def once(fd): + self.backend.volume_uuid.side_effect = None + self.backend.volume_uuid.return_value = UUID + return replace_root(fd) + self.backend.volume_uuid.side_effect = once + with self.assertRaisesRegex(ValueError, "root_changed"): + IDENTITY.capture_fd(self.fd) + self.assert_owned_fds_closed() + + def test_target_permissions_changed_during_observation(self): + def chmod_target(fd): + self.file.chmod(0o600) + return UUID + + self.file.chmod(0o640) + self.backend.volume_uuid.side_effect = chmod_target + with self.assertRaisesRegex(ValueError, "target_changed"): + IDENTITY.capture_fd(self.fd) + self.assert_owned_fds_closed() + + def test_target_fd_replacement_detected_while_retained_fd_remains_original(self): + other = self.root / "other" + other.write_bytes(b"different") + + def replace_target(fd): + with other.open("rb") as stream: + os.dup2(stream.fileno(), self.fd) + return UUID + + self.backend.volume_uuid.side_effect = replace_target + with self.assertRaisesRegex(ValueError, "target_changed"): + IDENTITY.capture_fd(self.fd) + self.assert_owned_fds_closed() + + def test_volume_uuid_drift_detected(self): + self.backend.volume_uuid.side_effect = [UUID, OTHER_UUID] + with self.assertRaisesRegex(ValueError, "volume_changed"): + IDENTITY.capture_fd(self.fd) + self.assert_owned_fds_closed() + + def test_filesystem_drift_detected_after_uuid_query(self): + self.backend.filesystem.side_effect = [self.fs, self.fs, + dataclasses.replace(self.fs, source=b"/dev/other")] + with self.assertRaisesRegex(ValueError, "filesystem_changed"): + IDENTITY.capture_fd(self.fd) + self.assert_owned_fds_closed() + + def test_metadata_syscall_error_closes_owned_fds(self): + self.backend.volume_uuid.side_effect = OSError("fixture failure") + with self.assertRaisesRegex(ValueError, "identity_unavailable"): + IDENTITY.capture_fd(self.fd) + self.assert_owned_fds_closed() + + def test_append_during_observation_does_not_change_identity(self): + def append(fd): + with self.file.open("ab") as stream: + stream.write(b"new transcript event\n") + return UUID + + self.backend.volume_uuid.side_effect = append + self.assertEqual(IDENTITY.capture_fd(self.fd)["inode"], os.fstat(self.fd).st_ino) + + def test_device_renumber_between_operations_is_not_durable(self): + before = IDENTITY.capture_fd(self.fd) + fstat, lstat = os.fstat, os.lstat + + def renumber(value): + fields = list(value) + fields[2] += 500 + return os.stat_result(fields) + + with mock.patch.object(IDENTITY.os, "fstat", side_effect=lambda fd: renumber(fstat(fd))), \ + mock.patch.object(IDENTITY.os, "lstat", side_effect=lambda *a, **k: renumber(lstat(*a, **k))): + after = IDENTITY.capture_fd(self.fd) + self.assertEqual(before, after) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tests/codex_plugin/test_hook_status.py b/Tests/codex_plugin/test_hook_status.py index cc34c08..fee4d49 100644 --- a/Tests/codex_plugin/test_hook_status.py +++ b/Tests/codex_plugin/test_hook_status.py @@ -116,6 +116,30 @@ def test_enrollment_and_policy_do_not_imply_completion(self): self.assertEqual(result["learner"]["latest_run"]["read_status"], "not_linked") self.assertNotIn("/private", json.dumps(result)) + def test_v2_identity_and_migration_hold_are_distinct_from_write_gate(self): + identity = {"scheme": "macos_volume_uuid_inode_v1", "volume_uuid": OTHER, "inode": 123} + self.write(self.modern / "learner/admission.json", {"enabled": True, + "mode": "host_sessions_v2", "schema_version": 2}) + self.write(self.modern / "learner/enrollments" / (SID + ".json"), + {"session_id": SID, "identity": identity, "transcript_path": "/private"}) + self.write(self.modern / "learner/migration-holds" / (SID + ".json"), + {"session_id": SID, "kind": "identity_migration_hold", "plan_sha256": DIGEST, + "private_note": "never display"}) + result = self.report()["learner"] + self.assertEqual(result["policy"]["metadata"]["mode"], "host_sessions_v2") + self.assertEqual(result["policy"]["metadata"]["schema_version"], 2) + self.assertEqual(result["enrollment"]["metadata"]["identity"], identity) + self.assertEqual(result["migration_hold"]["metadata"]["kind"], "identity_migration_hold") + self.assertNotIn("reconciliation_required", result["migration_hold"]["metadata"]) + self.assertNotIn("never display", json.dumps(result)) + self.assertNotIn("/private", json.dumps(result)) + + def test_stable_identity_errors_use_exact_allowlist(self): + for reason in ("admission_legacy_migration_required", "admission_migration_held", + "admission_identity_volume_changed", "admission_identity_filesystem_unsupported"): + self.assertEqual(STATUS.metadata({"reason": reason}), {"reason": reason}) + self.assertEqual(STATUS.metadata({"reason": "admission_identity_private_path_or_secret"}), {}) + def test_successful_run_retains_uuid_and_evidence_qualification(self): self.write(self.modern / "learner/events.jsonl", [{"session_id": SID, "run_id": RUN, "event": "learner_finished", "status": "succeeded"}], log=True) diff --git a/Tests/codex_plugin/test_host_v2.py b/Tests/codex_plugin/test_host_v2.py new file mode 100644 index 0000000..abca2d4 --- /dev/null +++ b/Tests/codex_plugin/test_host_v2.py @@ -0,0 +1,376 @@ +"""V2 admission fixtures; native identity calls are replaced by an explicit oracle. + +These tests qualify binding/lifecycle logic, not Darwin volume persistence. The +real extractor and owned-volume remount qualification have independent tests. +""" +import contextlib +import copy +import importlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +import test_frontier as F + +HOST = TRANSCRIPT = IDENTITY = None +VOLUME = '11111111-2222-4333-8444-555555555555' +OTHER_VOLUME = '22222222-3333-4444-8555-666666666666' + + +def setUpModule(): + global HOST, TRANSCRIPT, IDENTITY + F.setUpModule() + HOST = importlib.import_module(F.PACKAGE + '.host_admission') + TRANSCRIPT = importlib.import_module(F.PACKAGE + '.transcript') + IDENTITY = importlib.import_module(F.PACKAGE + '.file_identity') + + +def tearDownModule(): + F.tearDownModule() + + +class DeviceNumber: + def __init__(self, value, delta): + self.value, self.delta = value, delta + + def __getattr__(self, name): + return getattr(self.value, name) + self.delta if name == 'st_dev' else getattr(self.value, name) + + +class HostV2Tests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory(prefix='v2-admission-') + self.addCleanup(temp.cleanup) + self.fixture = F.Fixture(Path(temp.name).resolve()) + self.f = self.fixture + self.capture = mock.patch.object(IDENTITY, 'capture_fd', side_effect=self.identity) + self.capture.start() + self.addCleanup(self.capture.stop) + self.policy = {'schema_version': 2, 'mode': HOST.MODE_V2, 'enabled': True, + 'activation_id': F.ACTIVATION, 'cutoff': F.CUTOFF, + 'state_dir': str(self.f.root), + 'sessions_dir': F.ADMISSION._directory_binding(self.f.sessions, stable_identity=True)} + self.save_policy() + + def identity(self, fd): + return {'scheme': IDENTITY.SCHEME, 'volume_uuid': VOLUME, 'inode': os.fstat(fd).st_ino} + + def save_policy(self): + F.write_json(self.f.policy_path, self.policy) + + def enroll(self, event='SessionStart', **changes): + return HOST.observe(self.f.root, {**self.f.payload, 'hook_event_name': event, **changes}) + + def entry_path(self): + return self.f.root / 'enrollments' / (self.f.sid + '.json') + + def entry(self): + return json.loads(self.entry_path().read_text()) + + def request(self): + value = self.entry() + return {'session_id': self.f.sid, 'hook_session_id': self.f.sid, + 'event': 'Stop', 'turn_id': F.TURN2, 'cwd': str(self.f.project), + 'hook_cwd': str(self.f.project), 'transcript_path': str(self.f.path), + 'identity': value['identity']} + + def hold(self, kind='file'): + parent = F.private_directory(self.f.root / 'migration-holds') + path = parent / (self.f.sid + '.json') + if kind == 'file': + F.write_json(path, {'opaque': True}) + elif kind == 'symlink': + path.symlink_to(self.f.root / 'missing-target') + elif kind == 'directory': + path.mkdir(mode=0o700) + return path + + @contextlib.contextmanager + def renumbered(self): + original_fstat, original_stat = os.fstat, Path.stat + with mock.patch.object(os, 'fstat', side_effect=lambda fd: DeviceNumber(original_fstat(fd), 2)), \ + mock.patch.object(Path, 'stat', autospec=True, + side_effect=lambda path, **kwargs: DeviceNumber(original_stat(path, **kwargs), 2)): + yield + + def test_v2_origin_and_binding_have_only_stable_identity(self): + self.assertEqual(self.enroll(), (self.f.sid, True, False)) + entry = self.entry() + self.assertEqual(set(entry['project']), {'path', 'identity'}) + self.assertIn('identity', entry) + self.assertNotIn('device', entry) + self.assertNotIn('inode', entry) + binding = F.ADMISSION.check(self.f.root, self.request()) + self.assertEqual(binding['mode'], HOST.MODE_V2) + self.assertEqual(F.ADMISSION.enrollment_ids(self.f.root, self.policy), [self.f.sid]) + + def test_v2_accepts_device_renumbering_without_recapture(self): + self.enroll() + before = self.entry_path().read_bytes() + binding_before = HOST.check(self.f.root, self.request()) + self.f.append_turn(F.TURN2, 'New fact after the captured frontier.') + with self.renumbered(): + self.assertEqual(self.enroll(), (self.f.sid, False, False)) + self.assertEqual(HOST.check(self.f.root, self.request()), binding_before) + parsed = TRANSCRIPT.read_excerpt(self.f.path, binding_before['frontier_offset'], stable_identity=True) + self.assertEqual(parsed.metadata.identity, binding_before['identity']) + self.assertEqual(self.entry_path().read_bytes(), before) + self.assertIn('New fact after', parsed.text) + + def test_v1_still_refuses_device_renumbering(self): + self.policy.update(schema_version=1, mode=HOST.MODE, + sessions_dir=F.Fixture.binding(self.f.sessions)) + self.save_policy() + self.enroll() + before = self.entry_path().read_bytes() + with self.renumbered(), self.assertRaisesRegex(ValueError, '^admission_directory_identity_changed$'): + self.enroll() + self.assertEqual(self.entry_path().read_bytes(), before) + + def test_policy_schema_mode_and_directory_shapes_are_paired_strictly(self): + original = copy.deepcopy(self.policy) + candidates = [dict(original, schema_version=1), dict(original, schema_version=True), + dict(original, mode=HOST.MODE), + dict(original, sessions_dir=F.Fixture.binding(self.f.sessions)), + dict(original, schema_version=1, mode=HOST.MODE)] + for candidate in candidates: + with self.subTest(candidate=candidate): + self.policy = candidate + self.save_policy() + with self.assertRaises(ValueError): + HOST.policy(self.f.root) + + def test_legacy_entry_under_v2_requires_migration_without_reset(self): + stable_policy = copy.deepcopy(self.policy) + self.policy.update(schema_version=1, mode=HOST.MODE, + sessions_dir=F.Fixture.binding(self.f.sessions)) + self.save_policy() + self.enroll() + before = self.entry_path().read_bytes() + self.policy = stable_policy + self.save_policy() + self.f.append_turn(F.TURN2, 'Pending legacy content must remain pending.') + with self.assertRaisesRegex(ValueError, '^admission_legacy_migration_required$'): + self.enroll() + self.assertEqual(self.entry_path().read_bytes(), before) + + def test_different_uuid_same_inode_and_device_refused(self): + self.enroll() + def other(fd): + return dict(self.identity(fd), volume_uuid=OTHER_VOLUME) + with mock.patch.object(IDENTITY, 'capture_fd', side_effect=other), \ + self.assertRaisesRegex(ValueError, '^admission_directory_identity_changed$'): + self.enroll() + + def test_changed_transcript_uuid_with_unchanged_directories_refused(self): + self.enroll() + inode = self.f.path.stat().st_ino + def other(fd): + value = self.identity(fd) + return dict(value, volume_uuid=OTHER_VOLUME) if value['inode'] == inode else value + with mock.patch.object(IDENTITY, 'capture_fd', side_effect=other), \ + self.assertRaisesRegex(ValueError, '^admission_enrollment_origin_changed$'): + self.enroll() + + def test_replaced_transcript_inode_refused(self): + self.enroll() + replacement = self.f.path.with_suffix('.replacement') + replacement.write_bytes(self.f.path.read_bytes()) + os.replace(replacement, self.f.path) + with self.assertRaisesRegex(ValueError, '^admission_enrollment_origin_changed$'): + self.enroll() + + def test_initial_metadata_change_refused(self): + self.enroll() + self.f.meta['payload']['cli_version'] = 'mutated-version' + self.f.rewrite_initial() + with self.assertRaisesRegex(ValueError, '^admission_enrollment_origin_changed$'): + self.enroll() + + def test_frontier_content_change_refused(self): + self.enroll() + self.f.path.write_bytes(self.f.path.read_bytes().replace(b'example marker is amber', b'example marker is azure')) + with self.assertRaisesRegex(ValueError, '^admission_frontier_anchor_changed$'): + self.enroll() + + def test_frontier_truncation_refused(self): + self.enroll() + self.f.path.write_bytes(self.f.path.read_bytes().split(b'\n', 1)[0] + b'\n') + with self.assertRaisesRegex(ValueError, '^admission_frontier_truncated_or_invalid$'): + self.enroll() + + def test_frontier_rechecks_initial_metadata_on_its_actual_fd(self): + self.enroll() + origin = HOST._origin(self.policy, self.f.payload) + self.f.meta['payload']['cli_version'] = 'mutated-version' + self.f.rewrite_initial() + with self.assertRaisesRegex(ValueError, '^admission_initial_metadata_changed$'): + F.ADMISSION._frontier(origin) + + def test_hold_file_symlink_or_directory_prevents_learning_preserves_entry(self): + self.enroll() + before = self.entry_path().read_bytes() + for kind in ('file', 'symlink', 'directory'): + with self.subTest(kind=kind): + path = self.hold(kind) + with self.assertRaisesRegex(ValueError, '^admission_migration_held$'): + HOST.check(self.f.root, self.request()) + self.assertEqual(self.enroll(), (self.f.sid, False, False)) + self.assertEqual(self.entry_path().read_bytes(), before) + path.rmdir() if kind == 'directory' else path.unlink() + + def test_hold_does_not_create_missing_enrollment(self): + self.hold() + with self.assertRaisesRegex(ValueError, '^admission_migration_held$'): + self.enroll() + self.assertFalse(self.entry_path().exists()) + + def test_unsafe_hold_directory_refuses_even_if_target_absent(self): + self.enroll() + parent = self.f.root / 'migration-holds' + parent.symlink_to(self.f.root / 'absent') + with self.assertRaisesRegex(ValueError, '^admission_migration_held$'): + HOST.check(self.f.root, self.request()) + parent.unlink() + parent.mkdir(mode=0o755) + with self.assertRaisesRegex(ValueError, '^admission_migration_held$'): + F.ADMISSION.ensure_no_migration_hold(self.f.root, self.f.sid) + + def test_missing_enrollment_with_state_is_never_recaptured(self): + self.enroll() + self.entry_path().unlink() + F.write_json(self.f.state, {'offset': 333, 'reconciliation_required': {'reason': 'retained'}}) + before = self.f.state.read_bytes() + with self.assertRaisesRegex(ValueError, '^admission_unowned_existing_state$'): + self.enroll() + self.assertEqual(self.f.state.read_bytes(), before) + self.assertFalse(self.entry_path().exists()) + + def test_request_rejects_legacy_numbers_and_malformed_identity(self): + self.enroll() + for changed in ({'device': 0}, {'inode': 1}, {'identity': None}, + {'identity': {'scheme': IDENTITY.SCHEME, 'volume_uuid': VOLUME, 'inode': True}}): + with self.subTest(changed=changed), self.assertRaises(ValueError): + HOST.check(self.f.root, {**self.request(), **changed}) + + def test_manual_precompact_still_refused(self): + with self.assertRaisesRegex(ValueError, '^admission_manual_compaction_not_enabled$'): + self.enroll(event='PreCompact', trigger='manual') + self.assertFalse(self.entry_path().exists()) + + def make_child(self, *, known=True): + parent = F.uuid7(F.ORIGIN, suffix=900) + meta = self.f.meta['payload'] + meta.update(session_id=parent, source={'subagent': {'thread_spawn': {'parent_thread_id': parent}}}) + if known: + meta['subagent_history_start_ordinal'] = 10 + self.f.rewrite_initial() + self.f.payload.update(session_id=parent, agent_id=self.f.sid) + # The inherited content is ordinal 1; the new message belongs to child 11. + rows = self.f.path.read_bytes().splitlines(keepends=True) + first = rows[0] + self.f.path.write_bytes(first) + for ordinal, text in ((1, 'Inherited parent fact'), (11, 'New child fact')): + with self.f.path.open('a') as stream: + stream.write(json.dumps({'type': 'response_item', 'ordinal': ordinal, + 'payload': {'type': 'message', 'role': 'user', 'content': text}}) + '\n') + return len(first) + + def test_fresh_v2_child_preserves_known_inherited_boundary(self): + boundary = self.make_child() + self.assertEqual(self.enroll(event='SubagentStop'), (self.f.sid, True, True)) + self.assertEqual(self.entry()['frontier_offset'], boundary) + excerpt = TRANSCRIPT.read_excerpt(self.f.path, boundary, stable_identity=True) + self.assertIn('New child fact', excerpt.text) + self.assertNotIn('Inherited parent', excerpt.text) + + def test_old_v2_child_baselines_without_backfill(self): + self.make_child() + self.policy['cutoff'] = '2026-01-03T00:00:00Z' + self.save_policy() + self.assertEqual(self.enroll(event='SubagentStop'), (self.f.sid, True, False)) + self.assertEqual(self.entry()['frontier_offset'], self.f.path.stat().st_size) + + def test_unknown_v2_child_boundary_refuses_without_enrollment(self): + self.make_child(known=False) + with self.assertRaisesRegex(ValueError, '^admission_unknown_inherited_boundary$'): + self.enroll(event='SubagentStop') + self.assertFalse(self.entry_path().exists()) + + def test_v2_origin_hash_and_semantics_share_one_metadata_read(self): + # V2 origin parses the exact sealed line, rather than opening the path + # again to interpret its session/parent metadata. + with mock.patch.object(HOST, 'inspect_rollout', side_effect=AssertionError('unexpected second metadata read')): + origin = HOST._origin(self.policy, self.f.payload) + self.assertEqual(origin['identity']['inode'], self.f.path.stat().st_ino) + + def test_frontier_rechecks_identity_after_reading(self): + self.enroll() + origin = HOST._origin(self.policy, self.f.payload) + count = [] + def capture(fd): + count.append(fd) + value = self.identity(fd) + return value if len(count) == 1 else dict(value, volume_uuid=OTHER_VOLUME) + with mock.patch.object(IDENTITY, 'capture_fd', side_effect=capture), \ + self.assertRaisesRegex(ValueError, '^admission_transcript_identity_changed$'): + F.ADMISSION._frontier(origin) + + def test_migration_hold_preserves_gate_cursor_and_pending_pause(self): + self.enroll() + F.write_json(self.f.state, {'offset': 333, 'frontier_offset': 100, + 'reconciliation_required': {'reason': 'uncertain_write'}, 'recent_messages': ['sealed']}) + F.write_json(self.f.pending, {'request_id': 'retained', 'paused_request_id': 'retained', + 'pause_reason': 'batch_limit'}) + before = self.f.state.read_bytes(), self.f.pending.read_bytes(), self.entry_path().read_bytes() + self.hold() + self.assertEqual(self.enroll(), (self.f.sid, False, False)) + with self.assertRaisesRegex(ValueError, '^admission_migration_held$'): + HOST.check(self.f.root, self.request()) + self.assertEqual((self.f.state.read_bytes(), self.f.pending.read_bytes(), + self.entry_path().read_bytes()), before) + + def test_stable_transcript_identity_comes_from_read_fd(self): + called = [] + def capture(fd): + called.append(fd) + self.assertEqual(os.fstat(fd).st_ino, self.f.path.stat().st_ino) + return self.identity(fd) + with mock.patch.object(IDENTITY, 'capture_fd', side_effect=capture): + excerpt = TRANSCRIPT.read_excerpt(self.f.path, stable_identity=True) + self.assertTrue(called) + self.assertEqual(len(set(called)), 1) + self.assertEqual(excerpt.metadata.identity['inode'], self.f.path.stat().st_ino) + self.assertIn('example marker', excerpt.text) + + def test_stable_transcript_refuses_path_replacement_during_read(self): + real_visible = TRANSCRIPT._visible_message + replaced = [] + def visible(*args): + if not replaced: + replaced.append(True) + replacement = self.f.path.with_suffix('.replacement') + replacement.write_bytes(self.f.path.read_bytes()) + os.replace(replacement, self.f.path) + return real_visible(*args) + with mock.patch.object(TRANSCRIPT, '_visible_message', side_effect=visible), \ + self.assertRaisesRegex(ValueError, '^admission_transcript_identity_changed$'): + TRANSCRIPT.read_excerpt(self.f.path, stable_identity=True) + + def test_v1_transcript_never_captures_persistent_identity(self): + with mock.patch.object(IDENTITY, 'capture_fd', side_effect=AssertionError('v1 must not capture UUID')): + self.assertIsNone(TRANSCRIPT.inspect_rollout(self.f.path).identity) + self.assertIsNone(TRANSCRIPT.read_excerpt(self.f.path).metadata.identity) + + def test_stable_transcript_refuses_symlink(self): + alias = self.f.path.with_suffix('.alias') + alias.symlink_to(self.f.path) + with self.assertRaisesRegex(ValueError, '^admission_noncanonical_path$'): + TRANSCRIPT.inspect_rollout(alias, stable_identity=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tests/codex_plugin/test_migration.py b/Tests/codex_plugin/test_migration.py new file mode 100644 index 0000000..a3370f8 --- /dev/null +++ b/Tests/codex_plugin/test_migration.py @@ -0,0 +1,670 @@ +"""Owned fixture-only migration/recovery tests; process/native/store effects blocked.""" +import fcntl +import importlib +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import sys +import types +import unittest +from unittest import mock + +import test_frontier as F + +M = None + + +def setUpModule(): + global M + F.setUpModule() + M = importlib.import_module(F.PACKAGE + ".migration") + + +def tearDownModule(): + F.tearDownModule() + + +class MigrationFixture: + def __init__(self, base): + self.base = base + self.root = F.private_directory(base / "state") + self.sessions = F.private_directory(base / "rollouts") + self.project = F.private_directory(base / "project") + for name in ("enrollments", "sessions", "admissions", "pending"): + F.private_directory(self.root / name) + for path in (self.root / "worker.lock", self.root / "enqueue.lock", base / "learner-provider.lock"): + path.touch(mode=0o600) + self.policy = {"schema_version": 1, "mode": "host_sessions_v1", "enabled": True, + "activation_id": F.ACTIVATION, "cutoff": F.CUTOFF, + "state_dir": str(self.root), "sessions_dir": F.Fixture.binding(self.sessions)} + F.write_json(self.root / "admission.json", self.policy) + self.route = base / "routes.json" + F.write_json(self.route, {"schema_version": 1, "mode": "host_sessions_v1", "enabled": True, "state_dir": str(self.root)}) + self.tasks = {} + + def task(self, n=1, *, cursor=True, pending=True, paused=True, gated=True): + sid = F.uuid7(F.ORIGIN, suffix=n) + folder = F.private_directory(self.sessions / "2026/01/02") + path = folder / ("rollout-2026-01-02T00-00-00-" + sid + ".jsonl") + meta = {"type": "session_meta", "ordinal": 0, "payload": { + "id": sid, "timestamp": F.CREATED, "cwd": str(self.project), "source": "vscode", "cli_version": "fixture"}} + path.write_text(json.dumps(meta) + "\n") + payload = {"session_id": sid, "transcript_path": str(path), "cwd": str(self.project), "agent_id": sid} + origin = M.host_admission._origin(self.policy, payload) + frontier = M.admission._frontier(origin) + entry = {**origin, **frontier, "activation_id": F.ACTIVATION, "captured_at": "2026-01-02T00:00:02Z"} + F.write_json(self.root / "enrollments" / (sid + ".json"), entry) + with path.open("a") as stream: + stream.write(json.dumps({"type": "response_item", "ordinal": 1, "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "fixture durable fact"}]}}) + "\n") + binding = M._binding(entry, self.policy, (self.root / "admission.json").read_bytes()) + state = {"session_id": sid, "transcript_path": str(path), "device": entry["device"], "inode": entry["inode"], + "offset": frontier["frontier_offset"], "admission": binding, "recent_messages": ["saved-dedup"], + "last_excerpt_sha256": "a" * 64, "current_turn_id": "turn-one", "status": "failed"} + if gated: + state["reconciliation_required"] = {"run_id": "historical-unknown", "memory_ids": ["unaltered-id"]} + request = {"session_id": sid, "hook_session_id": sid, "transcript_path": str(path), "device": entry["device"], + "inode": entry["inode"], "cwd": str(self.project), "hook_cwd": str(self.project), "admission": binding, + "request_id": "fixture-request-" + str(n), "event": "Stop", "turn_id": "turn-one", "size_bytes": path.stat().st_size} + if paused: + request.update(paused_request_id=request["request_id"], pause_reason="reconciliation_required", backoff_until="2099-01-01") + record = {"binding": binding, "state_sha256": M.object_digest(state), "last_event": "e" * 64, "processing": None} + if cursor: + F.write_json(self.root / "sessions" / (sid + ".json"), state) + F.write_json(self.root / "admissions" / (sid + ".json"), record) + if pending: + F.write_json(self.root / "pending" / (sid + ".json"), request) + self.tasks[sid] = {"entry": entry, "state": state, "record": record, "request": request, "path": path} + return sid + + +class MigrationTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.base = Path(temporary.name).resolve() + self.fixture = MigrationFixture(self.base) + self.root, self.route = self.fixture.root, self.fixture.route + self.uuid = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + capture = mock.patch.object(M.file_identity, "capture_fd", side_effect=lambda fd: { + "scheme": "macos_volume_uuid_inode_v1", "volume_uuid": self.uuid, "inode": os.fstat(fd).st_ino}) + capture.start() + self.addCleanup(capture.stop) + self.sid = self.fixture.task() + + def prepare(self, ids=None, **kwargs): + return M.prepare(self.root, self.route, ids or [self.sid], authorize_current_volume_adoption=True, **kwargs) + + def apply(self, plan): + return M.apply(self.root, plan, M.digest(M.encode(plan))) + + def snapshot(self): + return {str(p.relative_to(self.base)): p.read_bytes() for p in self.base.rglob("*") if p.is_file()} + + def test_plan_is_read_only_and_records_absence(self): + sid2 = self.fixture.task(2, cursor=False, pending=False) + before = self.snapshot() + plan = self.prepare([self.sid, sid2]) + self.assertEqual(before, self.snapshot()) + self.assertFalse(plan["historical_volume_continuity_proven"]) + self.assertTrue(plan["owner_authorized_current_volume_adoption"]) + absent = [r for r in plan["records"] if r["preimage"] is None] + self.assertEqual(len(absent), 3) + + def test_explicit_authorization_and_bounded_selection(self): + with self.assertRaisesRegex(ValueError, "explicit_identity_adoption_authorization_required"): + M.prepare(self.root, self.route, [self.sid]) + with self.assertRaisesRegex(ValueError, "invalid_selection"): + self.prepare([self.sid, self.sid]) + with self.assertRaisesRegex(ValueError, "invalid_selection"): + self.prepare([F.uuid7(F.ORIGIN, suffix=n) for n in range(33)]) + + def test_apply_preserves_every_nonidentity_field_and_other_task(self): + other = self.fixture.task(2) + before_other = {name: path.read_bytes() for name, path in M._paths(self.root, other).items()} + plan = self.prepare() + result = self.apply(plan) + self.assertEqual(result["status"], "migration_complete_held") + self.assertFalse(result["learner_started"]) + for name, path in M._paths(self.root, other).items(): + self.assertEqual(before_other[name], path.read_bytes()) + old = self.fixture.tasks[self.sid] + for name, key in (("sessions", "state"), ("pending", "request")): + actual = json.loads(M._paths(self.root, self.sid)[name].read_bytes()) + original = old[key] + self.assertEqual({k: v for k, v in actual.items() if k not in {"identity", "admission"}}, + {k: v for k, v in original.items() if k not in {"device", "inode", "admission"}}) + record = json.loads(M._paths(self.root, self.sid)["admissions"].read_bytes()) + self.assertEqual(record["last_event"], old["record"]["last_event"]) + self.assertEqual(record["legacy_event_identity"], {"device": old["entry"]["device"], "inode": old["entry"]["inode"]}) + request = json.loads(M._paths(self.root, self.sid)["pending"].read_bytes()) + with self.assertRaisesRegex(ValueError, "migration_held"): + M.host_admission.check(self.root, request) + + def test_previously_unpaused_request_is_ineligible_until_explicit_release(self): + sid = self.fixture.task(2, paused=False, gated=False) + plan = self.prepare([sid]) + self.apply(plan) + request = json.loads(M._paths(self.root, sid)["pending"].read_bytes()) + self.assertNotIn("paused_request_id", request) + with self.assertRaisesRegex(ValueError, "migration_held"): + M.host_admission.check(self.root, request) + sha = M.digest(M.encode(plan)) + release = M.prepare_release(self.root, plan, sha, [sid]) + before = {p: p.read_bytes() for p in M._paths(self.root, sid).values()} + result = M.release(self.root, plan, sha, release, M.digest(M.encode(release))) + self.assertFalse(result["learner_started"]) + self.assertEqual(before, {p: p.read_bytes() for p in before}) + self.assertEqual(M.host_admission.check(self.root, request), request["admission"]) + + def test_disabled_route_and_policy_stay_disabled(self): + policy = {**self.fixture.policy, "enabled": False} + F.write_json(self.root / "admission.json", policy) + F.write_json(self.route, {"schema_version": 1, "mode": "host_sessions_v1", "enabled": False, "state_dir": str(self.root)}) + # Rebuild this selected chain against the intentionally disabled policy. + old = self.fixture.tasks[self.sid] + binding = M._binding(old["entry"], policy, (self.root / "admission.json").read_bytes()) + state = {**old["state"], "admission": binding} + F.write_json(M._paths(self.root, self.sid)["sessions"], state) + F.write_json(M._paths(self.root, self.sid)["admissions"], {**old["record"], "binding": binding, "state_sha256": M.object_digest(state)}) + F.write_json(M._paths(self.root, self.sid)["pending"], {**old["request"], "admission": binding}) + self.apply(self.prepare()) + self.assertIs(json.loads((self.root / "admission.json").read_bytes())["enabled"], False) + self.assertIs(json.loads(self.route.read_bytes())["enabled"], False) + + def test_later_selected_migration_retains_v2_policy_bytes_and_digest(self): + second = self.fixture.task(2) + first = self.prepare() + self.apply(first) + policy = (self.root / "admission.json").read_bytes() + legacy = self.root / "migrations" / M.digest(M.encode(first)) / "legacy-policy.json" + later = self.prepare([second], legacy_policy_path=legacy) + self.apply(later) + self.assertEqual((self.root / "admission.json").read_bytes(), policy) + self.assertEqual(M._unb64(later["records"][-1]["candidate"]), policy) + + def test_no_pending_and_no_cursor_are_supported_without_creation(self): + sid = self.fixture.task(2, cursor=False, pending=False) + plan = self.prepare([sid]) + self.apply(plan) + for name in ("admissions", "sessions", "pending"): + self.assertFalse(M._paths(self.root, sid)[name].exists()) + + def test_no_pending_existing_cursor_is_preserved(self): + sid = self.fixture.task(2, pending=False) + self.apply(self.prepare([sid])) + self.assertFalse(M._paths(self.root, sid)["pending"].exists()) + self.assertTrue(M._paths(self.root, sid)["sessions"].exists()) + + def test_legacy_device_drift_is_explicitly_adopted_but_inode_change_is_rejected(self): + # Rewrite only historical device values, consistently across legacy chain. + for path in [self.root / "admission.json", *M._paths(self.root, self.sid).values()]: + value = json.loads(path.read_bytes()) + def walk(item): + if isinstance(item, dict): + for k, v in item.items(): + if k == "device": item[k] = v + 111 + else: walk(v) + walk(value) + F.write_json(path, value) + self._repair_fixture_chain() + self.apply(self.prepare()) + + def _repair_fixture_chain(self): + paths = M._paths(self.root, self.sid) + policy = json.loads((self.root / "admission.json").read_bytes()) + entry = json.loads(paths["enrollments"].read_bytes()) + binding = M._binding(entry, policy, (self.root / "admission.json").read_bytes()) + state = json.loads(paths["sessions"].read_bytes()) + state["admission"] = binding + F.write_json(paths["sessions"], state) + record = json.loads(paths["admissions"].read_bytes()) + record.update(binding=binding, state_sha256=M.object_digest(state)) + F.write_json(paths["admissions"], record) + request = json.loads(paths["pending"].read_bytes()) + request["admission"] = binding + F.write_json(paths["pending"], request) + + def test_altered_inode_metadata_or_frontier_is_rejected(self): + path = M._paths(self.root, self.sid)["enrollments"] + original = path.read_bytes() + for key, value in (("inode", 1), ("initial_meta_sha256", "f" * 64), ("frontier_anchor_sha256", "f" * 64)): + with self.subTest(key=key): + entry = json.loads(original) + entry[key] = value + F.write_json(path, entry) + with self.assertRaises(ValueError): self.prepare() + path.write_bytes(original) + + def test_active_processing_and_state_digest_mismatch_refuse(self): + path = M._paths(self.root, self.sid)["admissions"] + original = json.loads(path.read_bytes()) + for change in ({"processing": "running-request"}, {"state_sha256": "0" * 64}): + F.write_json(path, {**original, **change}) + with self.assertRaises(ValueError): self.prepare() + + def test_stale_cas_refuses_before_journal_or_hold(self): + plan = self.prepare() + path = M._paths(self.root, self.sid)["pending"] + value = json.loads(path.read_bytes()) + F.write_json(path, {**value, "newer": True}) + with self.assertRaisesRegex(ValueError, "cas_mismatch"): + self.apply(plan) + self.assertFalse((self.root / "migrations").exists()) + self.assertFalse((self.root / "migration-holds").exists()) + + def test_absent_record_creation_after_plan_refuses(self): + sid = self.fixture.task(2, cursor=False, pending=False) + plan = self.prepare([sid]) + F.write_json(M._paths(self.root, sid)["pending"], {"unexpected": True}) + with self.assertRaises(ValueError): self.apply(plan) + + def test_symlink_and_nonprivate_preimage_refuse(self): + path = M._paths(self.root, self.sid)["pending"] + original = path.read_bytes() + path.chmod(0o644) + with self.assertRaisesRegex(ValueError, "unsafe_file"): self.prepare() + path.unlink() + replacement = self.base / "other.json" + replacement.write_bytes(original) + replacement.chmod(0o600) + path.symlink_to(replacement) + with self.assertRaisesRegex(ValueError, "unsafe_file"): self.prepare() + + def test_plan_candidate_tampering_is_recomputed_and_refused(self): + plan = self.prepare() + record = plan["records"][1] + value = M.admission._json(M._unb64(record["candidate"])) + value.pop("reconciliation_required") + raw = M.encode(value) + record.update(candidate=M._b64(raw), candidate_sha256=M.digest(raw)) + with self.assertRaisesRegex(ValueError, "candidate_transformation_invalid"): + self.apply(plan) + + def test_different_current_volume_same_inode_refuses_reviewed_plan(self): + plan = self.prepare() + self.uuid = "bbbbbbbb-bbbb-4ccc-8ddd-eeeeeeeeeeee" + with self.assertRaises(ValueError): self.apply(plan) + + def test_runtime_change_refuses(self): + plan = self.prepare() + with mock.patch.object(M, "_runtime", return_value={"different": "0" * 64}): + with self.assertRaisesRegex(ValueError, "runtime_changed"): self.apply(plan) + + def test_partial_every_publication_recovers_without_unpausing(self): + for cut in ("transitional-policy", 0, 1, 2, 3, 4, "final-policy"): + with self.subTest(cut=cut): + other_base = F.private_directory(self.base / ("partial-" + str(cut))) + fixture = MigrationFixture(other_base) + sid = fixture.task(paused=False) + plan = M.prepare(fixture.root, fixture.route, [sid], authorize_current_volume_adoption=True) + sha = M.digest(M.encode(plan)) + real = M._publish + def crash(record, work, index): + real(record, work, index) + if index == cut: + raise RuntimeError("injected interruption after atomic rename") + with mock.patch.object(M, "_publish", side_effect=crash): + with self.assertRaisesRegex(RuntimeError, "injected interruption"): + M.apply(fixture.root, plan, sha) + hold = fixture.root / "migration-holds" / (sid + ".json") + self.assertEqual(hold.exists(), cut != "transitional-policy") + result = M.recover(fixture.root, plan, sha) + self.assertEqual(result["status"], "migration_complete_held") + M.recover(fixture.root, plan, sha) # Idempotent exact completed journal. + state = json.loads(M._paths(fixture.root, sid)["sessions"].read_bytes()) + self.assertEqual(state["reconciliation_required"], fixture.tasks[sid]["state"]["reconciliation_required"]) + + def test_partial_recovery_never_overwrites_newer_state(self): + plan = self.prepare() + real = M._publish + def crash(record, work, index): + real(record, work, index) + raise RuntimeError("stop") + with mock.patch.object(M, "_publish", side_effect=crash): + with self.assertRaises(RuntimeError): self.apply(plan) + path = M._paths(self.root, self.sid)["sessions"] + F.write_json(path, {"newer": "must survive"}) + newer = path.read_bytes() + with self.assertRaisesRegex(ValueError, "cas_mismatch"): + M.recover(self.root, plan, M.digest(M.encode(plan))) + self.assertEqual(path.read_bytes(), newer) + + def test_nonprefix_candidates_are_rejected(self): + plan = self.prepare() + target = plan["records"][2] + Path(target["path"]).write_bytes(M._unb64(target["candidate"])) + with self.assertRaisesRegex(ValueError, "nonprefix_publication"): + M.recover(self.root, plan, M.digest(M.encode(plan))) + + def test_hold_replacement_or_chain_change_blocks_release(self): + plan = self.prepare() + self.apply(plan) + sha = M.digest(M.encode(plan)) + release = M.prepare_release(self.root, plan, sha, [self.sid]) + hold = self.root / "migration-holds" / (self.sid + ".json") + hold.write_bytes(hold.read_bytes()) + with self.assertRaisesRegex(ValueError, "release_hold_cas_mismatch"): + M.release(self.root, plan, sha, release, M.digest(M.encode(release))) + path = M._paths(self.root, self.sid)["sessions"] + value = json.loads(path.read_bytes()) + F.write_json(path, {**value, "offset": value["offset"] + 1}) + with self.assertRaisesRegex(ValueError, "completed_chain_changed"): + M.prepare_release(self.root, plan, sha, [self.sid]) + + def test_matching_release_is_idempotent_and_does_not_retry(self): + plan = self.prepare() + self.apply(plan) + sha = M.digest(M.encode(plan)) + release = M.prepare_release(self.root, plan, sha, [self.sid]) + rsha = M.digest(M.encode(release)) + M.release(self.root, plan, sha, release, rsha) + result = M.release(self.root, plan, sha, release, rsha) + self.assertFalse(result["learner_started"]) + + def test_changed_journal_or_missing_hold_refuses_recovery(self): + plan = self.prepare() + self.apply(plan) + sha = M.digest(M.encode(plan)) + hold = self.root / "migration-holds" / (self.sid + ".json") + hold.unlink() + with self.assertRaisesRegex(ValueError, "hold_missing_after_publication"): + M.recover(self.root, plan, sha) + + def test_real_lock_contention_releases_earlier_locks_without_writing(self): + for busy in (self.root / "worker.lock", self.base / "learner-provider.lock", self.root / "enqueue.lock"): + with self.subTest(busy=busy.name): + fd = os.open(busy, os.O_RDONLY) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + before = self.snapshot() + with self.assertRaises(BlockingIOError): self.prepare() + self.assertEqual(before, self.snapshot()) + finally: + os.close(fd) + with M.Locks(self.root): + pass + + def test_changed_lock_identity_is_detected(self): + with M.Locks(self.root) as locks: + path = self.root / "worker.lock" + path.unlink() + path.touch(mode=0o600) + with self.assertRaisesRegex(ValueError, "lock_changed"): + locks.verify() + + def test_release_selected_subsets_preserves_sibling_hold(self): + second = self.fixture.task(2) + plan = self.prepare([self.sid, second]) + self.apply(plan) + sha = M.digest(M.encode(plan)) + for sid in (self.sid, second): + release = M.prepare_release(self.root, plan, sha, [sid]) + M.release(self.root, plan, sha, release, M.digest(M.encode(release))) + self.assertFalse((self.root / "migration-holds" / (sid + ".json")).exists()) + if sid == self.sid: + self.assertTrue((self.root / "migration-holds" / (second + ".json")).exists()) + + def test_partial_release_resumes_only_matching_intent(self): + second = self.fixture.task(2) + plan = self.prepare([self.sid, second]) + self.apply(plan) + sha = M.digest(M.encode(plan)) + release = M.prepare_release(self.root, plan, sha, [self.sid, second]) + rsha = M.digest(M.encode(release)) + original = M._event + def stop(work, label, **fields): + if label == "hold_released": + raise RuntimeError("crash after unlink before journal receipt") + original(work, label, **fields) + with mock.patch.object(M, "_event", side_effect=stop): + with self.assertRaisesRegex(RuntimeError, "crash after unlink"): + M.release(self.root, plan, sha, release, rsha) + first_hold = self.root / "migration-holds" / (self.sid + ".json") + second_hold = self.root / "migration-holds" / (second + ".json") + self.assertFalse(first_hold.exists()) + self.assertTrue(second_hold.exists()) + M.release(self.root, plan, sha, release, rsha) + self.assertFalse(second_hold.exists()) + + def test_missing_or_tampered_journal_is_not_recreated_by_recover(self): + plan = self.prepare() + sha = M.digest(M.encode(plan)) + with self.assertRaises((ValueError, FileNotFoundError)): + M.recover(self.root, plan, sha) + self.apply(plan) + journal = self.root / "migrations" / sha / "legacy-policy.json" + journal.write_text("{}\n") + with self.assertRaisesRegex(ValueError, "journal_changed"): + M.recover(self.root, plan, sha) + + def test_later_migration_requires_retained_policy_not_supplied_lookalike(self): + second = self.fixture.task(2) + first = self.prepare() + self.apply(first) + fake = self.base / "lookalike.json" + fake.write_bytes(M._unb64(first["legacy_policy"]["bytes"])) + fake.chmod(0o600) + with self.assertRaisesRegex(ValueError, "legacy_policy_not_retained_journal"): + self.prepare([second], legacy_policy_path=fake) + + def test_later_migration_pins_original_journal_source(self): + second = self.fixture.task(2) + first = self.prepare() + self.apply(first) + original = self.root / "migrations" / M.digest(M.encode(first)) / "legacy-policy.json" + later = self.prepare([second], legacy_policy_path=original) + original.write_bytes(original.read_bytes()) # Same bytes, newer preimage identity. + with self.assertRaisesRegex(ValueError, "legacy_source_changed"): + self.apply(later) + + def test_hold_first_failure_recovers_without_metadata_change(self): + plan = self.prepare() + sha = M.digest(M.encode(plan)) + original = M._event + def fail(work, label, **fields): + if label == "hold_published": raise RuntimeError("hold interrupted") + original(work, label, **fields) + with mock.patch.object(M, "_event", side_effect=fail): + with self.assertRaises(RuntimeError): self.apply(plan) + for record in plan["records"]: + expected = plan["transitional_policy"]["candidate"] if record["path"] == str(self.root / "admission.json") else record["preimage"] + self.assertEqual(Path(record["path"]).read_bytes(), M._unb64(expected)) + M.recover(self.root, plan, sha) + + def test_legacy_event_compatibility_is_absent_when_no_last_event(self): + path = M._paths(self.root, self.sid)["admissions"] + record = json.loads(path.read_bytes()) + record["last_event"] = None + F.write_json(path, record) + self.apply(self.prepare()) + self.assertNotIn("legacy_event_identity", json.loads(path.read_bytes())) + + def test_foreign_activation_and_route_root_are_refused(self): + original = self.route.read_bytes() + route = json.loads(original) + route["state_dir"] = str(self.base) + F.write_json(self.route, route) + with self.assertRaisesRegex(ValueError, "invalid_route"): self.prepare() + self.route.write_bytes(original) + path = M._paths(self.root, self.sid)["enrollments"] + entry = json.loads(path.read_bytes()) + entry["activation_id"] = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + F.write_json(path, entry) + with self.assertRaisesRegex(ValueError, "legacy_enrollment_invalid"): self.prepare() + + def test_legacy_directory_inode_change_refuses(self): + policy = json.loads((self.root / "admission.json").read_bytes()) + policy["sessions_dir"]["inode"] += 1 + F.write_json(self.root / "admission.json", policy) + with self.assertRaisesRegex(ValueError, "legacy_directory_inode_changed"): self.prepare() + + def test_output_and_input_plan_bytes_are_private_exact_and_exclusive(self): + plan = self.prepare() + path = self.base / "plan.json" + raw = M.encode(plan) + M._new(path, raw) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual(M._load_plan(path, M.digest(raw)), plan) + with self.assertRaises(FileExistsError): M._new(path, raw) + with self.assertRaisesRegex(ValueError, "plan_file_sha_mismatch"): + M._load_plan(path, "0" * 64) + + def test_cli_deadline_restores_signal_handlers_on_refusal(self): + import signal + handlers = {n: signal.getsignal(n) for n in (signal.SIGALRM, signal.SIGTERM, signal.SIGINT)} + with mock.patch.object(M, "prepare", side_effect=ValueError("migration_fixture_refusal")), mock.patch("builtins.print"): + result = M.main(["plan", "--state-dir", str(self.root), "--routes", str(self.route), + "--session-id", self.sid, "--output", str(self.base / "unused.json")]) + self.assertEqual(result, 1) + self.assertEqual(signal.getitimer(signal.ITIMER_REAL), (0.0, 0.0)) + self.assertEqual({n: signal.getsignal(n) for n in handlers}, handlers) + + def _advance_released_task(self, sid, *, processing=None, remove_pending=False, new_gate=False): + paths = M._paths(self.root, sid) + state = json.loads(paths["sessions"].read_bytes()) + state.update(offset=Path(state["transcript_path"]).stat().st_size, status="succeeded", + recent_messages=["newer-message"], last_excerpt_sha256="b" * 64) + state.pop("reconciliation_required", None) + if new_gate: + state["reconciliation_required"] = {"run_id": "newer-uncertain-run", "memory_ids": ["newer-unverified-id"]} + F.write_json(paths["sessions"], state) + record = json.loads(paths["admissions"].read_bytes()) + record.update(state_sha256=M.object_digest(state), processing=processing, last_event="d" * 64) + record.pop("legacy_event_identity", None) + F.write_json(paths["admissions"], record) + if remove_pending: + paths["pending"].unlink() + else: + pending = json.loads(paths["pending"].read_bytes()) + pending.update(request_id="newer-request", paused_request_id="newer-request", pause_reason="newer-hold") + F.write_json(paths["pending"], pending) + return {name: path.read_bytes() if path.exists() else None for name, path in paths.items()} + + def test_sequential_release_after_sibling_progress_keeps_newer_bytes(self): + second = self.fixture.task(2) + plan = self.prepare([self.sid, second]) + self.apply(plan) + sha = M.digest(M.encode(plan)) + first_release = M.prepare_release(self.root, plan, sha, [self.sid]) + M.release(self.root, plan, sha, first_release, M.digest(M.encode(first_release))) + advanced = self._advance_released_task(self.sid, remove_pending=True) + second_release = M.prepare_release(self.root, plan, sha, [second]) + M.release(self.root, plan, sha, second_release, M.digest(M.encode(second_release))) + self.assertFalse((self.root / "migration-holds" / (second + ".json")).exists()) + self.assertEqual(advanced, {n: p.read_bytes() if p.exists() else None for n, p in M._paths(self.root, self.sid).items()}) + + def test_interrupted_release_after_sibling_progress_preserves_processing_and_new_gate(self): + second = self.fixture.task(2) + plan = self.prepare([self.sid, second]) + self.apply(plan) + sha = M.digest(M.encode(plan)) + release = M.prepare_release(self.root, plan, sha, [self.sid, second]) + rsha = M.digest(M.encode(release)) + original = M._event + def stop(work, label, **fields): + if label == "hold_released": raise RuntimeError("partial release") + original(work, label, **fields) + with mock.patch.object(M, "_event", side_effect=stop): + with self.assertRaises(RuntimeError): M.release(self.root, plan, sha, release, rsha) + advanced = self._advance_released_task(self.sid, processing="newer-interrupted-run", new_gate=True) + M.release(self.root, plan, sha, release, rsha) + self.assertFalse((self.root / "migration-holds" / (second + ".json")).exists()) + self.assertEqual(advanced, {n: p.read_bytes() if p.exists() else None for n, p in M._paths(self.root, self.sid).items()}) + + def test_released_bad_digest_or_regressed_cursor_blocks_sibling_release(self): + second = self.fixture.task(2) + plan = self.prepare([self.sid, second]) + self.apply(plan) + sha = M.digest(M.encode(plan)) + release = M.prepare_release(self.root, plan, sha, [self.sid]) + M.release(self.root, plan, sha, release, M.digest(M.encode(release))) + self._advance_released_task(self.sid) + path = M._paths(self.root, self.sid)["sessions"] + state = json.loads(path.read_bytes()) + F.write_json(path, {**state, "offset": 1}) + with self.assertRaisesRegex(ValueError, "released_cursor_chain_invalid"): + M.prepare_release(self.root, plan, sha, [second]) + record_path = M._paths(self.root, self.sid)["admissions"] + record = json.loads(record_path.read_bytes()) + F.write_json(record_path, {**record, "state_sha256": M.object_digest(json.loads(path.read_bytes()))}) + with self.assertRaisesRegex(ValueError, "released_cursor_regressed_or_truncated"): + M.prepare_release(self.root, plan, sha, [second]) + + def _legacy_admission(self): + base = Path(__file__).parent / "fixtures/identity_v1" + manifest = json.loads((base / "SOURCE.json").read_bytes()) + name = "_migration_retained_v1" + package = types.ModuleType(name) + package.__path__ = [str(base)] + sys.modules[name] = package + sys.modules[name + ".transcript"] = importlib.import_module(F.PACKAGE + ".transcript") + self.addCleanup(lambda: [sys.modules.pop(k) for k in list(sys.modules) if k == name or k.startswith(name + ".")]) + for filename, expected in manifest["files"].items(): + self.assertEqual(M.digest((base / filename).read_bytes()), expected) + return importlib.import_module(name + ".admission") + + def test_transitional_policy_refuses_actual_retained_v1_and_current_v2_workers(self): + unselected = self.fixture.task(2, paused=False, gated=False) + plan = self.prepare() + legacy = self._legacy_admission() + self.assertEqual(legacy.check_activation(self.root)["mode"], "host_sessions_v1") + sha = M.digest(M.encode(plan)) + original = M._event + def stop(work, label, **fields): + if label == "transitional_policy_published": raise RuntimeError("transition crash") + original(work, label, **fields) + with mock.patch.object(M, "_event", side_effect=stop): + with self.assertRaises(RuntimeError): M.apply(self.root, plan, sha) + for module in (legacy, M.admission): + with self.assertRaisesRegex(ValueError, "admission_inactive"): + module.check_activation(self.root) + before = {p: p.read_bytes() for p in M._paths(self.root, unselected).values()} + with mock.patch.object(F.RUNNER, "spawn_worker") as spawn, mock.patch.object(F.RUNNER, "process_request") as process: + # The actual worker rejects before inspecting the unpaused queue. + with self.assertRaisesRegex(ValueError, "admission_inactive"): + F.RUNNER.worker(self.root) + spawn.assert_not_called() + process.assert_not_called() + self.assertEqual(before, {p: p.read_bytes() for p in before}) + self.assertFalse((self.root / "migration-holds").exists()) + M.recover(self.root, plan, sha) + + def test_transition_and_each_hold_journal_phase_can_recover(self): + for label in ("before_transitional_policy", "transitional_policy_published", "hold_published", "before_final_policy", "final_policy_published"): + with self.subTest(label=label): + base = F.private_directory(self.base / label) + fixture = MigrationFixture(base) + first, second = fixture.task(), fixture.task(2) + plan = M.prepare(fixture.root, fixture.route, [first, second], authorize_current_volume_adoption=True) + sha = M.digest(M.encode(plan)) + original = M._event + count = 0 + def stop(work, observed, **fields): + nonlocal count + original(work, observed, **fields) + if observed == label: + count += 1 + raise RuntimeError("phase interruption") + with mock.patch.object(M, "_event", side_effect=stop): + with self.assertRaises(RuntimeError): M.apply(fixture.root, plan, sha) + self.assertEqual(count, 1) + M.recover(fixture.root, plan, sha) + self.assertEqual(json.loads((fixture.root / "admission.json").read_bytes())["enabled"], True) + + def test_early_final_policy_or_unknown_transition_is_refused(self): + plan = self.prepare() + path = self.root / "admission.json" + path.write_bytes(M._unb64(plan["records"][-1]["candidate"])) + with self.assertRaisesRegex(ValueError, "final_policy_published_before_records"): + M.recover(self.root, plan, M.digest(M.encode(plan))) + value = json.loads(path.read_bytes()) + value["activation_id"] = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + F.write_json(path, value) + with self.assertRaisesRegex(ValueError, "policy_cas_mismatch"): + M.recover(self.root, plan, M.digest(M.encode(plan))) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tests/codex_plugin/test_v2_runtime.py b/Tests/codex_plugin/test_v2_runtime.py new file mode 100644 index 0000000..9280849 --- /dev/null +++ b/Tests/codex_plugin/test_v2_runtime.py @@ -0,0 +1,175 @@ +"""Stable identity in queue/cursor processing; providers remain in-process fakes.""" +from dataclasses import replace +import contextlib +import json +import os +import unittest +from unittest import mock + +import test_host as H + + +def setUpModule(): + H.setUpModule() + + +def tearDownModule(): + H.tearDownModule() + + +class StableRuntimeTests(unittest.TestCase): + def setUp(self): + self.f = H.HostTests() + self.f.setUp() + self.addCleanup(self.f.doCleanups) + self.runner = H.F.RUNNER + self.identity = H.F.ADMISSION.file_identity + self.volume = "11111111-2222-4333-8444-555555555555" + patch = mock.patch.object(self.identity, "capture_fd", side_effect=self.capture) + patch.start(); self.addCleanup(patch.stop) + self.f.policy.update(schema_version=2, mode="host_sessions_v2", + sessions_dir=H.F.ADMISSION._directory_binding(self.f.sessions, stable_identity=True)) + H.F.write_json(self.f.root / "admission.json", self.f.policy) + H.F.write_json(self.f.routes, {"schema_version": 2, "mode": "host_sessions_v2", + "enabled": True, "state_dir": str(self.f.root)}) + + def capture(self, fd): + return {"scheme": "macos_volume_uuid_inode_v1", "volume_uuid": self.volume, + "inode": os.fstat(fd).st_ino} + + def queued(self): + sid, path, payload = self.f.task() + self.assertFalse(self.f.dispatch(payload, "SessionStart")) + self.f.append(path, "A new durable fixture fact after the enrollment frontier.") + self.assertTrue(self.f.dispatch(payload)) + return sid, path, payload + + def test_queue_and_completed_cursor_have_only_stable_identity(self): + sid, _, _ = self.queued() + request = self.f.request(sid) + self.assertEqual(request["identity"]["volume_uuid"], self.volume) + self.assertNotIn("device", request) + self.assertNotIn("inode", request) + self.assertEqual(self.f.process(sid), "succeeded") + state = json.loads((self.f.root / "sessions" / (sid + ".json")).read_bytes()) + self.assertEqual(state["identity"], request["identity"]) + self.assertNotIn("device", state) + self.assertNotIn("inode", state) + self.assertGreater(state["offset"], request["admission"]["frontier_offset"]) + + def test_later_device_renumbering_does_not_reject_pending_request(self): + sid, _, _ = self.queued() + original_inspect, original_read = self.runner.inspect_rollout, self.runner.read_excerpt + def inspect(*args, **kwargs): + value = original_inspect(*args, **kwargs) + return replace(value, device=value.device + 100) + def read(*args, **kwargs): + value = original_read(*args, **kwargs) + return replace(value, metadata=replace(value.metadata, device=value.metadata.device + 100)) + with mock.patch.object(self.runner, "inspect_rollout", side_effect=inspect), \ + mock.patch.object(self.runner, "read_excerpt", side_effect=read): + self.assertEqual(self.f.process(sid), "succeeded") + self.assertEqual(len(self.f.invocations), 1) + + def test_same_inode_on_different_volume_is_rejected_before_provider(self): + sid, path, _ = self.queued() + inode = path.stat().st_ino + def changed(fd): + value = self.capture(fd) + if value["inode"] == inode: + value["volume_uuid"] = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + return value + with mock.patch.object(self.identity, "capture_fd", side_effect=changed): + with self.assertRaisesRegex(ValueError, "admission_enrollment_origin_changed"): + self.f.process(sid) + self.assertEqual(self.f.invocations, []) + + def test_identity_change_between_inspection_and_excerpt_rejects(self): + sid, _, _ = self.queued() + original_read = self.runner.read_excerpt + def changed(*args, **kwargs): + value = original_read(*args, **kwargs) + identity = {**value.metadata.identity, "volume_uuid": "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"} + return replace(value, metadata=replace(value.metadata, identity=identity)) + with mock.patch.object(self.runner, "read_excerpt", side_effect=changed): + with self.assertRaisesRegex(ValueError, "transcript_replaced_during_read"): + self.f.process(sid) + self.assertEqual(self.f.invocations, []) + + def test_legacy_last_event_stays_deduplicated_without_pending(self): + sid, _, payload = self.queued() + request = self.f.request(sid) + self.assertEqual(self.f.process(sid), "succeeded") + record_path = self.f.root / "admissions" / (sid + ".json") + record = json.loads(record_path.read_bytes()) + historical = {"device": 99, "inode": request["identity"]["inode"]} + fields = {key: request.get(key) for key in ("event", "turn_id", "trigger", "size_bytes")} + record.update(last_event=self.runner.admission_digest({**fields, **historical}), + legacy_event_identity=historical) + H.F.write_json(record_path, record) + pending = self.f.root / "pending" / (sid + ".json") + pending.unlink() + before = record_path.read_bytes() + self.assertFalse(self.f.dispatch(payload)) + self.assertFalse(pending.exists()) + self.assertEqual(record_path.read_bytes(), before) + self.assertTrue(self.f.dispatch(payload, turn_id="a-new-turn")) + record = json.loads(record_path.read_bytes()) + self.assertNotIn("legacy_event_identity", record) + self.assertEqual(record["last_event"], self.runner.event_digest(self.f.request(sid))) + + def test_partial_route_policy_migration_never_dispatches(self): + _, _, payload = self.f.task() + H.F.write_json(self.f.routes, {"schema_version": 1, "mode": "host_sessions_v1", + "enabled": True, "state_dir": str(self.f.root)}) + self.assertFalse(self.f.dispatch(payload, "SessionStart")) + self.assertEqual(self.f.receipt()["reason"], "admission_route_mismatch") + self.assertFalse((self.f.root / "enrollments").exists()) + + def test_inflight_legacy_hook_rechecks_route_after_waiting_for_enrollment_lock(self): + self.f.policy.update(schema_version=1, mode="host_sessions_v1", + sessions_dir=H.F.Fixture.binding(self.f.sessions)) + H.F.write_json(self.f.root / "admission.json", self.f.policy) + route = {"schema_version": 1, "mode": "host_sessions_v1", "enabled": True, + "state_dir": str(self.f.root)} + H.F.write_json(self.f.routes, route) + _, _, payload = self.f.task() + original = self.runner.lock_file + @contextlib.contextmanager + def after_migration(path, *args, **kwargs): + H.F.write_json(self.f.routes, {**route, "schema_version": 2, "mode": "host_sessions_v2"}) + with original(path, *args, **kwargs) as held: + yield held + with mock.patch.object(self.runner, "lock_file", side_effect=after_migration): + self.assertFalse(self.f.dispatch(payload, "SessionStart")) + self.assertEqual(self.f.receipt()["reason"], "admission_route_mismatch") + self.assertFalse((self.f.root / "enrollments").exists()) + self.assertFalse((self.f.root / "pending").exists()) + + def test_route_changes_after_observation_before_queue_leave_task_bytes_unchanged(self): + sid, path, payload = self.queued() + protected = [self.f.root / directory / (sid + ".json") + for directory in ("enrollments", "sessions", "admissions", "pending")] + before = {path: path.read_bytes() for path in protected} + self.f.append(path, "Another appended fact at the new event.", ordinal=12) + original = self.runner.lock_file + acquired = 0 + @contextlib.contextmanager + def after_observation(path, *args, **kwargs): + nonlocal acquired + acquired += 1 + if acquired == 2: + H.F.write_json(self.f.routes, {"schema_version": 1, "mode": "host_sessions_v1", + "enabled": True, "state_dir": str(self.f.root)}) + with original(path, *args, **kwargs) as held: + yield held + with mock.patch.object(self.runner, "lock_file", side_effect=after_observation): + self.assertFalse(self.f.dispatch(payload, turn_id="new-turn")) + self.assertEqual(acquired, 2) + self.assertEqual(before, {path: path.read_bytes() for path in protected}) + events = [json.loads(line) for line in (self.f.root / "events.jsonl").read_text().splitlines()] + self.assertEqual(events[-1]["reason"], "admission_route_mismatch") + + +if __name__ == "__main__": + unittest.main() diff --git a/codex/plugins/engram/README.md b/codex/plugins/engram/README.md index 4ecb45d..4aeedfd 100644 --- a/codex/plugins/engram/README.md +++ b/codex/plugins/engram/README.md @@ -80,6 +80,37 @@ apply at Codex's supported reload boundary; existing-task pickup needs verificat ## Migration +### Persistent transcript identity + +Fresh host setup uses `host_sessions_v2`: a local APFS volume UUID plus inode +identifies each bound directory and transcript. A device number may change after +a remount; it is checked within each read for races, but is not saved as the v2 +identity. Unsupported filesystems fail closed. Existing host settings are kept +unchanged when the package is updated. + +Existing `host_sessions_v1` tasks require explicit, selected-task migration with +`scripts/codex_learner_migrate.py`. Its `plan` command validates the original +policy, enrollment frontier, transcript metadata, cursor and pending request; +the resulting plan binds exact preimages and runtime hashes. Inspect that plan +before passing its path and SHA256 to `apply`. When no historical volume UUID +exists, current-volume adoption must be explicitly authorized and is recorded +as such; migration does not prove historical volume continuity. + +Applying a plan temporarily disables host admission while publishing the selected +records and route, then restores the policy's original enabled setting in v2. +The journal supports `recover` after an interrupted publication. Other tasks' +records remain untouched; unmigrated v1 enrollments under v2 require their own +migration. Later plans use the retained original v1 policy as their legacy anchor. + +Each migrated task receives a separate migration hold. `release-plan` and +`release` remove only explicitly selected holds after validating their records. +They do not clear a reconciliation gate, change a cursor or paused request, retry +work, or launch a learner. Previously released tasks may advance normally while +their siblings remain held. Use the CLI's `--help` and `hook-status` to inspect +the required arguments and current state. + +### Previous installers + `engram@personal` succeeds `engram-hooks@personal`. Disable the old plugin through Codex's plugin controls before trusting the successor. Keep old cached resources for open tasks that still reference them. The new package never replays or deletes diff --git a/codex/plugins/engram/scripts/codex_learner/admission.py b/codex/plugins/engram/scripts/codex_learner/admission.py index b64efaf..c3709b9 100644 --- a/codex/plugins/engram/scripts/codex_learner/admission.py +++ b/codex/plugins/engram/scripts/codex_learner/admission.py @@ -12,6 +12,8 @@ import stat import uuid +from . import file_identity + MAX_BYTES = 1024 * 1024 LINEAGE_KEYS = ( "forked_from_id", "parent_thread_id", "forked_from_ordinal_exclusive", @@ -59,9 +61,40 @@ def _directory(path): return info -def _bound_directory(value): - _require(isinstance(value, dict) and set(value) == {"path", "device", "inode"}, +def _directory_binding(path, *, stable_identity=False): + """Capture an owned directory; durable v2 identity comes from its open FD.""" + path = _canonical(str(path)) + info = _directory(path) + if not stable_identity: + return {"path": str(path), "device": info.st_dev, "inode": info.st_ino} + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK | os.O_DIRECTORY + fd = os.open(path, flags) + try: + opened = os.fstat(fd) + _require(stat.S_ISDIR(opened.st_mode) and opened.st_uid == os.getuid() + and (opened.st_dev, opened.st_ino) == (info.st_dev, info.st_ino), + "directory_identity_changed") + identity = file_identity.capture_fd(fd) + after = _directory(_canonical(str(path))) + final = os.fstat(fd) + _require((after.st_dev, after.st_ino, after.st_uid, after.st_mode) + == (final.st_dev, final.st_ino, final.st_uid, final.st_mode) + == (opened.st_dev, opened.st_ino, opened.st_uid, opened.st_mode), + "directory_identity_changed") + return {"path": str(path), "identity": identity} + finally: + os.close(fd) + + +def _bound_directory(value, *, stable_identity=False): + fields = {"path", "identity"} if stable_identity else {"path", "device", "inode"} + _require(isinstance(value, dict) and set(value) == fields, "invalid_directory_binding") + if stable_identity: + expected = file_identity.validate(value["identity"]) + actual = _directory_binding(value["path"], stable_identity=True) + _require(actual["identity"] == expected, "directory_identity_changed") + return Path(actual["path"]) path = _canonical(value["path"]) info = _directory(path) _require(all(type(value[key]) is int and value[key] >= 0 for key in ("device", "inode")) @@ -70,7 +103,7 @@ def _bound_directory(value): return path -def _owned_bytes(path, *, first_line=False, private=False): +def _owned_bytes_record(path, *, first_line=False, private=False, stable_identity=False): flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK with os.fdopen(os.open(path, flags), "rb") as stream: info = os.fstat(stream.fileno()) @@ -78,9 +111,24 @@ def _owned_bytes(path, *, first_line=False, private=False): "file_not_owned_regular") _require(not private or not info.st_mode & 0o022, "policy_writable_by_others") _require(first_line or info.st_size <= MAX_BYTES, "record_too_large") + identity = file_identity.capture_fd(stream.fileno()) if stable_identity else None raw = stream.readline(MAX_BYTES + 1) if first_line else stream.read(MAX_BYTES + 1) + if stable_identity: + _require(file_identity.capture_fd(stream.fileno()) == identity, + "transcript_identity_changed") + after = os.fstat(stream.fileno()) + current = _canonical(str(path)).stat(follow_symlinks=False) + _require((after.st_dev, after.st_ino, after.st_uid, after.st_mode) + == (info.st_dev, info.st_ino, info.st_uid, info.st_mode) + == (current.st_dev, current.st_ino, current.st_uid, current.st_mode), + "transcript_identity_changed") _require(len(raw) <= MAX_BYTES, "record_too_large") _require(not first_line or raw.endswith(b"\n"), "initial_metadata_incomplete") + return raw, info, identity + + +def _owned_bytes(path, *, first_line=False, private=False): + raw, info, _ = _owned_bytes_record(path, first_line=first_line, private=private) return raw, info @@ -173,8 +221,22 @@ def _frontier(origin, offset=None): flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK with os.fdopen(os.open(origin["transcript_path"], flags), "rb") as stream: info = os.fstat(stream.fileno()) - _require(stat.S_ISREG(info.st_mode) and (info.st_dev, info.st_ino, info.st_uid) - == (origin["device"], origin["inode"], os.getuid()), "transcript_identity_changed") + stable = "identity" in origin + if stable: + _require("device" not in origin and "inode" not in origin, "invalid_transcript_binding") + identity = file_identity.validate(origin["identity"]) + _require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() + and file_identity.capture_fd(stream.fileno()) == identity, + "transcript_identity_changed") + _require(type(origin["initial_meta_bytes"]) is int + and 0 < origin["initial_meta_bytes"] <= MAX_BYTES, "invalid_initial_metadata_size") + initial = stream.read(origin["initial_meta_bytes"]) + _require(len(initial) == origin["initial_meta_bytes"] + and hashlib.sha256(initial).hexdigest() == origin["initial_meta_sha256"], + "initial_metadata_changed") + else: + _require(stat.S_ISREG(info.st_mode) and (info.st_dev, info.st_ino, info.st_uid) + == (origin["device"], origin["inode"], os.getuid()), "transcript_identity_changed") # Capture only the observed EOF. A partial tail is never rounded backwards. offset = info.st_size if offset is None else offset _require(type(offset) is int and origin["initial_meta_bytes"] <= offset <= info.st_size, @@ -183,6 +245,14 @@ def _frontier(origin, offset=None): stream.seek(start) data = stream.read(offset - start) _require(len(data) == offset - start and data.endswith(b"\n"), "frontier_not_complete_line") + if stable: + after = os.fstat(stream.fileno()) + current = _canonical(origin["transcript_path"]).stat(follow_symlinks=False) + _require(file_identity.capture_fd(stream.fileno()) == identity + and (after.st_dev, after.st_ino, after.st_uid, after.st_mode) + == (info.st_dev, info.st_ino, info.st_uid, info.st_mode) + == (current.st_dev, current.st_ino, current.st_uid, current.st_mode), + "transcript_identity_changed") return {"frontier_offset": offset, "frontier_anchor_start": start, "frontier_anchor_sha256": hashlib.sha256(data).hexdigest()} @@ -240,12 +310,17 @@ def check_activation(root: Path) -> dict: def enrollment_ids(root: Path, policy: dict) -> list[str]: - if policy.get("mode") == "host_sessions_v1": + if policy.get("mode") in {"host_sessions_v1", "host_sessions_v2"}: from .host_admission import enrollment_ids as host_ids return host_ids(root) return list(policy["enrollments"]) +def ensure_no_migration_hold(root: Path, sid: str): + from . import host_admission + host_admission.ensure_no_migration_hold(root, sid) + + def check(root: Path, request: dict) -> dict: from . import host_admission if host_admission.mode(root): diff --git a/codex/plugins/engram/scripts/codex_learner/file_identity.py b/codex/plugins/engram/scripts/codex_learner/file_identity.py new file mode 100644 index 0000000..d72fba4 --- /dev/null +++ b/codex/plugins/engram/scripts/codex_learner/file_identity.py @@ -0,0 +1,220 @@ +"""Descriptor-bound persistent identities for qualified local Darwin volumes. + +The ABI below follows Darwin SDK sys/mount.h, sys/attr.h and getattrlist(2). +Volume attributes must be queried on the volume root; ATTR_CMN_FILEID cannot be +combined with that query. Python's 64-bit fstat inode belongs to the retained +target descriptor instead. Device numbers are used only during this observation. + +Only local APFS is currently qualified. This is local filesystem provenance, +not authentication against an account owner or a cloned filesystem. Callers +still enforce canonical paths, ownership, metadata and transcript frontier seals. +""" + +from __future__ import annotations + +import ctypes +from dataclasses import dataclass +import os +import platform +import stat +import struct +import sys +import uuid +from typing import Any + + +SCHEME = "macos_volume_uuid_inode_v1" +_MAX_INODE = (1 << 64) - 1 +_MNT_LOCAL = 0x00001000 +_RETURNED = 0x80000000 +_VOL_INFO = 0x80000000 +_VOL_CAPABILITIES = 0x00020000 +_VOL_UUID = 0x00040000 +_VOLUME_MASK = _VOL_INFO | _VOL_CAPABILITIES | _VOL_UUID +_PERSISTENT_64 = 0x00000001 | 0x00020000 +_ATTR_SIZE = 4 + 20 + 32 + 16 + + +def _require(condition: bool, reason: str) -> None: + if not condition: + raise ValueError("admission_identity_" + reason) + + +def validate(value: Any) -> dict[str, Any]: + """Validate the exact durable schema; never coerce or repair stored values.""" + _require(type(value) is dict and set(value) == {"scheme", "volume_uuid", "inode"}, + "invalid") + _require(value["scheme"] == SCHEME, "scheme_unsupported") + raw = value["volume_uuid"] + _require(type(raw) is str and len(raw) == 36, "uuid_invalid") + try: + parsed = uuid.UUID(raw) + except (ValueError, AttributeError): + raise ValueError("admission_identity_uuid_invalid") from None + _require(parsed.int != 0 and str(parsed) == raw, "uuid_invalid") + _require(type(value["inode"]) is int and 0 < value["inode"] <= _MAX_INODE, + "inode_invalid") + return value + + +class _AttrList(ctypes.Structure): + _fields_ = [("bitmapcount", ctypes.c_uint16), ("reserved", ctypes.c_uint16), + ("commonattr", ctypes.c_uint32), ("volattr", ctypes.c_uint32), + ("dirattr", ctypes.c_uint32), ("fileattr", ctypes.c_uint32), + ("forkattr", ctypes.c_uint32)] + + +class _StatFS64(ctypes.Structure): + # __DARWIN_STRUCT_STATFS64, including the complete 1024-byte mount names. + _fields_ = [ + ("f_bsize", ctypes.c_uint32), ("f_iosize", ctypes.c_int32), + ("f_blocks", ctypes.c_uint64), ("f_bfree", ctypes.c_uint64), + ("f_bavail", ctypes.c_uint64), ("f_files", ctypes.c_uint64), + ("f_ffree", ctypes.c_uint64), ("f_fsid", ctypes.c_int32 * 2), + ("f_owner", ctypes.c_uint32), ("f_type", ctypes.c_uint32), + ("f_flags", ctypes.c_uint32), ("f_fssubtype", ctypes.c_uint32), + ("f_fstypename", ctypes.c_char * 16), + ("f_mntonname", ctypes.c_char * 1024), + ("f_mntfromname", ctypes.c_char * 1024), + ("f_flags_ext", ctypes.c_uint32), ("f_reserved", ctypes.c_uint32 * 7), + ] + + +@dataclass(frozen=True) +class _Filesystem: + fsid: tuple[int, int] + kind: int + subtype: int + flags: int + name: bytes + mount: str + source: bytes + + +def _c_string(value: _StatFS64, field: str, length: int) -> bytes: + raw = ctypes.string_at(ctypes.addressof(value) + getattr(_StatFS64, field).offset, + length) + _require(b"\0" in raw, "filesystem_malformed") + return raw.split(b"\0", 1)[0] + + +def _parse_volume(raw: bytes) -> str: + # getattrlist packs even 64-bit fields on 4-byte boundaries. All selected + # fields are fixed-size: length, attribute_set_t, capabilities/valid, UUID. + _require(len(raw) == _ATTR_SIZE, "attributes_malformed") + fields = struct.unpack("=14I16s", raw) + _require(fields[0] == _ATTR_SIZE, "attributes_malformed") + # ATTR_VOL_INFO is a request selector, not a returned attribute. + _require(fields[1:6] == (_RETURNED, _VOL_CAPABILITIES | _VOL_UUID, 0, 0, 0), + "attributes_unsupported") + capabilities, valid = fields[6:10], fields[10:14] + _require(capabilities[0] & _PERSISTENT_64 == _PERSISTENT_64 + and valid[0] & _PERSISTENT_64 == _PERSISTENT_64, + "persistence_unsupported") + identity = str(uuid.UUID(bytes=fields[14])) + _require(identity != "00000000-0000-0000-0000-000000000000", "uuid_invalid") + return identity + + +class _Darwin: + def __init__(self) -> None: + machine = platform.machine() + _require(sys.platform == "darwin" and machine in {"arm64", "x86_64"} + and ctypes.sizeof(ctypes.c_void_p) == 8, "platform_unsupported") + _require(ctypes.sizeof(_StatFS64) == 2168 + and _StatFS64.f_fsid.offset == 48 + and _StatFS64.f_mntonname.offset == 88 + and ctypes.sizeof(_AttrList) == 24, "abi_unsupported") + try: + self.lib = ctypes.CDLL("/usr/lib/libSystem.B.dylib", use_errno=True) + # sys/cdefs.h uses $INODE64 on x86_64, the unsuffixed 64-bit-only + # entry point on arm64. Never call the legacy x86_64 layout. + self.statfs = getattr(self.lib, "fstatfs$INODE64" if machine == "x86_64" + else "fstatfs") + self.statfs.argtypes = [ctypes.c_int, ctypes.POINTER(_StatFS64)] + self.statfs.restype = ctypes.c_int + self.getattr = self.lib.fgetattrlist + self.getattr.argtypes = [ctypes.c_int, ctypes.POINTER(_AttrList), + ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint] + self.getattr.restype = ctypes.c_int + except (OSError, AttributeError): + raise ValueError("admission_identity_api_unavailable") from None + + def filesystem(self, fd: int) -> _Filesystem: + value = _StatFS64() + _require(self.statfs(fd, ctypes.byref(value)) == 0, "filesystem_unavailable") + name = _c_string(value, "f_fstypename", 16) + _require(name == b"apfs" and value.f_flags & _MNT_LOCAL != 0, + "filesystem_unsupported") + mount = os.fsdecode(_c_string(value, "f_mntonname", 1024)) + _require(bool(mount) and os.path.isabs(mount) + and os.path.normpath(mount) == mount, "mount_invalid") + return _Filesystem(tuple(value.f_fsid), value.f_type, value.f_fssubtype, + value.f_flags, name, mount, + _c_string(value, "f_mntfromname", 1024)) + + def volume_uuid(self, fd: int) -> str: + request = _AttrList(5, 0, _RETURNED, _VOLUME_MASK, 0, 0, 0) + buffer = ctypes.create_string_buffer(_ATTR_SIZE) + # REPORT_FULLSIZE exposes truncation; PACK_INVAL_ATTRS gives a fixed + # layout whose returned masks must still show every requested value. + _require(self.getattr(fd, ctypes.byref(request), buffer, _ATTR_SIZE, + 0x00000004 | 0x00000008) == 0, + "attributes_unavailable") + return _parse_volume(buffer.raw) + + +def _descriptor_identity(value: os.stat_result) -> tuple[int, ...]: + return (value.st_dev, value.st_ino, value.st_mode, value.st_uid, value.st_gid, + value.st_nlink) + + +def capture_fd(fd: int) -> dict[str, Any]: + """Capture a stable identity without closing, reading or seeking caller FD. + + There are no retries, subprocesses or persistent caches. Filesystem metadata + is queried a fixed number of times. Existing hook process deadlines remain + responsible for bounding a kernel/filesystem stall, as with other fstat calls. + """ + _require(type(fd) is int and 0 <= fd <= (1 << 31) - 1, "fd_invalid") + backend = _Darwin() + retained = root = None + try: + initial = os.fstat(fd) + _require(stat.S_ISREG(initial.st_mode) or stat.S_ISDIR(initial.st_mode), + "file_type_unsupported") + _require(0 < initial.st_ino <= _MAX_INODE and initial.st_nlink > 0, + "inode_invalid") + target = _descriptor_identity(initial) + retained = os.dup(fd) + _require(_descriptor_identity(os.fstat(retained)) == target, "target_changed") + filesystem = backend.filesystem(retained) + mount = filesystem.mount + _require(os.path.realpath(mount, strict=True) == mount, "mount_invalid") + root = os.open(mount, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + | os.O_CLOEXEC | os.O_NONBLOCK) + root_info = os.fstat(root) + _require(stat.S_ISDIR(root_info.st_mode) and root_info.st_dev == initial.st_dev + and root_info.st_nlink > 0, "root_mismatch") + root_identity = _descriptor_identity(root_info) + _require(_descriptor_identity(os.lstat(mount)) == root_identity, + "root_changed") + _require(backend.filesystem(root) == filesystem, "root_mismatch") + volume_uuid = backend.volume_uuid(root) + _require(backend.volume_uuid(root) == volume_uuid, "volume_changed") + _require(backend.filesystem(retained) == filesystem + and backend.filesystem(root) == filesystem, "filesystem_changed") + _require(_descriptor_identity(os.fstat(retained)) == target + and _descriptor_identity(os.fstat(fd)) == target, "target_changed") + _require(_descriptor_identity(os.fstat(root)) == root_identity + and _descriptor_identity(os.lstat(mount)) == root_identity + and os.path.realpath(mount, strict=True) == mount, "root_changed") + return validate({"scheme": SCHEME, "volume_uuid": volume_uuid, + "inode": initial.st_ino}) + except OSError: + raise ValueError("admission_identity_unavailable") from None + finally: + if root is not None: + os.close(root) + if retained is not None: + os.close(retained) diff --git a/codex/plugins/engram/scripts/codex_learner/host_admission.py b/codex/plugins/engram/scripts/codex_learner/host_admission.py index e9c920b..e74c1f3 100644 --- a/codex/plugins/engram/scripts/codex_learner/host_admission.py +++ b/codex/plugins/engram/scripts/codex_learner/host_admission.py @@ -10,18 +10,25 @@ import os from pathlib import Path import re +import stat -from . import admission -from .transcript import inspect_rollout +from . import admission, file_identity +from .transcript import inspect_rollout, _metadata_from_line MODE = "host_sessions_v1" +MODE_V2 = "host_sessions_v2" +MODES = (MODE, MODE_V2) OBSERVE_EVENTS = {"SessionStart", "UserPromptSubmit", "SubagentStart", "SubagentStop", "Stop", "PreCompact", "SessionEnd"} LEARN_EVENTS = {"Stop", "SubagentStop", "PreCompact"} def mode(root): raw, _ = admission._owned_bytes(root / "admission.json", private=True) - return admission._json(raw).get("mode") == MODE + return admission._json(raw).get("mode") in MODES + + +def is_v2(value): + return value.get("mode") == MODE_V2 def policy(root): @@ -32,25 +39,27 @@ def policy(root): value = admission._json(raw) admission._require(set(value) == {"schema_version", "mode", "enabled", "activation_id", "cutoff", "state_dir", "sessions_dir"}, "invalid_host_policy_schema") - admission._require(type(value["schema_version"]) is int and value["schema_version"] == 1 - and value["mode"] == MODE, "unsupported_policy") + admission._require(type(value["schema_version"]) is int + and ((value["schema_version"], value["mode"]) in ((1, MODE), (2, MODE_V2))), + "unsupported_policy") admission._require(value["enabled"] is True, "inactive") admission._uuid(value["activation_id"], 4) admission._require(admission._timestamp(value["cutoff"]) <= datetime.now(timezone.utc), "future_cutoff") admission._require(value["state_dir"] == str(root), "state_dir_mismatch") - admission._bound_directory(value["sessions_dir"]) + admission._bound_directory(value["sessions_dir"], stable_identity=is_v2(value)) return value, raw def _origin(value, payload): sid = payload.get("session_id") path = admission._canonical(payload.get("transcript_path")) - sessions = admission._bound_directory(value["sessions_dir"]) + stable = is_v2(value) + sessions = admission._bound_directory(value["sessions_dir"], stable_identity=stable) admission._require(path.is_relative_to(sessions), "transcript_outside_sessions") parts = path.relative_to(sessions).parts admission._require(len(parts) == 4 and re.fullmatch(r"\d{4}/\d{2}/\d{2}", "/".join(parts[:3])), "invalid_rollout_layout") - raw, info = admission._owned_bytes(path, first_line=True) + raw, info, identity = admission._owned_bytes_record(path, first_line=True, stable_identity=stable) record = admission._json(raw) meta = record.get("payload") admission._require(record.get("type") == "session_meta" and isinstance(meta, dict), "initial_metadata_required") @@ -59,7 +68,10 @@ def _origin(value, payload): admission._require(re.fullmatch(r"rollout-" + re.escape("-".join(parts[:3])) + r"T\d{2}-\d{2}-\d{2}-" + re.escape(canonical_sid) + r"\.jsonl", parts[3]) is not None, "invalid_rollout_filename") - parsed = inspect_rollout(path) + # For v2, the metadata hash and fork-boundary interpretation use the very + # same bytes from the FD that supplied the persistent identity. + parsed = (_metadata_from_line(path, info, raw, len(raw), identity) + if stable else inspect_rollout(path)) admission._require(parsed.session_id == canonical_sid and parsed.fork_boundary_known, "unknown_inherited_boundary") if sid != canonical_sid: source = parsed.source if isinstance(parsed.source, dict) else {} @@ -74,15 +86,37 @@ def _origin(value, payload): version = meta.get("cli_version") admission._require(isinstance(version, str) and 0 < len(version) <= 128, "unqualified_version") project = admission._canonical(meta.get("cwd")) - project_info = admission._directory(project) + project_binding = admission._directory_binding(project, stable_identity=stable) admission._require(payload.get("cwd") == str(project), "project_mismatch") created = admission._timestamp(meta.get("timestamp")) admission._require(created <= datetime.now(timezone.utc), "invalid_origin_time") - return {"session_id": canonical_sid, "transcript_path": str(path), "device": info.st_dev, - "inode": info.st_ino, "uid": info.st_uid, "initial_meta_bytes": len(raw), + transcript_binding = {"identity": identity} if stable else {"device": info.st_dev, "inode": info.st_ino} + return {"session_id": canonical_sid, "transcript_path": str(path), **transcript_binding, + "uid": info.st_uid, "initial_meta_bytes": len(raw), "initial_meta_sha256": hashlib.sha256(raw).hexdigest(), "source": source, "cli_version": version, "origin_metadata_timestamp": admission._utc(created), - "project": {"path": str(project), "device": project_info.st_dev, "inode": project_info.st_ino}} + "project": project_binding} + + +def ensure_no_migration_hold(root, sid): + """Any marker or unsafe marker path holds learning; its contents grant nothing.""" + admission._uuid(sid, 7) + parent = root / "migration-holds" + try: + info = parent.lstat() + except FileNotFoundError: + return + except OSError as error: + raise ValueError("admission_migration_held") from error + admission._require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() + and not info.st_mode & 0o077, "migration_held") + try: + (parent / (sid + ".json")).lstat() + except FileNotFoundError: + return + except OSError as error: + raise ValueError("admission_migration_held") from error + raise ValueError("admission_migration_held") def _entry_path(root, sid): @@ -100,6 +134,8 @@ def _validate_entry(root, value, payload): origin = _origin(value, payload) raw, _ = admission._owned_bytes(_entry_path(root, origin["session_id"]), private=True) entry = admission._json(raw) + if is_v2(value) and "identity" not in entry and {"device", "inode"} <= set(entry): + raise ValueError("admission_legacy_migration_required") admission._require(set(entry) == set(origin) | admission.FRONTIER_FIELDS | {"captured_at", "activation_id"}, "invalid_enrollment_schema") admission._require(entry["activation_id"] == value["activation_id"], "enrollment_activation_changed") @@ -128,10 +164,13 @@ def observe(root, payload): if path.exists(): _validate_entry(root, value, payload) return origin["session_id"], False, False + ensure_no_migration_hold(root, origin["session_id"]) # Never adopt state from another admission or silently reset missing entries. for state_path in admission.state_paths(root, origin["session_id"]).values(): admission._require(not state_path.exists(), "unowned_existing_state") - parsed = inspect_rollout(origin["transcript_path"]) + parsed = inspect_rollout(origin["transcript_path"], stable_identity=is_v2(value)) + if is_v2(value): + admission._require(parsed.identity == origin["identity"], "transcript_identity_changed") # A newly spawned child can finish before any child-start hook exposes its # transcript. Its known inherited-history boundary keeps the initial visible # excerpt child-only, and the cutoff excludes pre-installation children. @@ -151,6 +190,7 @@ def observe(root, payload): def check(root, request): + ensure_no_migration_hold(root, request.get("session_id")) value, raw = policy(root) admission._require(request.get("event") in LEARN_EVENTS, "unqualified_event") if request["event"] == "Stop": @@ -163,9 +203,15 @@ def check(root, request): entry = _validate_entry(root, value, payload) admission._require(request.get("session_id") == entry["session_id"] and request.get("cwd") == entry["project"]["path"], "session_or_project_mismatch") - admission._require(all(request.get(key) == entry[key] for key in ("device", "inode")), "transcript_identity_changed") + if is_v2(value): + admission._require("device" not in request and "inode" not in request, + "invalid_transcript_binding") + admission._require(file_identity.validate(request.get("identity")) == entry["identity"], + "transcript_identity_changed") + else: + admission._require(all(request.get(key) == entry[key] for key in ("device", "inode")), "transcript_identity_changed") return {**entry, "cutoff": value["cutoff"], "policy_sha256": hashlib.sha256(raw).hexdigest(), - "state_dir": str(root), "mode": MODE} + "state_dir": str(root), "mode": value["mode"]} def enrollment_ids(root): diff --git a/codex/plugins/engram/scripts/codex_learner/migration.py b/codex/plugins/engram/scripts/codex_learner/migration.py new file mode 100644 index 0000000..7882bdd --- /dev/null +++ b/codex/plugins/engram/scripts/codex_learner/migration.py @@ -0,0 +1,856 @@ +"""Explicit, selected-task host identity migration; never enqueue or run a learner. + +Plans contain exact private preimages and candidates. A caller must review and pin +the plan SHA256 before apply/recover/release. Publication uses a durable journal, +a disabled transitional policy, holds, task records, route, then final policy. Recovery only +completes the same expected publication prefix; it never rolls back newer state. +Legacy volume continuity is deliberately NOT asserted by current UUID adoption. +""" +from __future__ import annotations + +import argparse +import base64 +import copy +from datetime import datetime, timezone +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import stat + +from . import admission, file_identity, host_admission + +MAX_SELECTED = 32 +MAX_PLAN_BYTES = 32 * 1024 * 1024 +MODE_V1 = "host_sessions_v1" +MODE_V2 = "host_sessions_v2" +PLAN_KIND = "selected_host_identity_migration" +HOLD_KIND = "identity_migration_hold" +ORDER = "transitional_policy_then_holds_then_selected_records_then_route_then_final_policy" + + +def require(ok, reason): + if not ok: + raise ValueError("migration_" + reason) + + +def encode(value): + return (json.dumps(value, sort_keys=True, separators=(",", ":"), + ensure_ascii=False, allow_nan=False) + "\n").encode() + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def object_digest(value): + # Match runner.admission_digest without importing the worker/provider module. + return digest(encode(value)[:-1]) + + +def _b64(raw): + return None if raw is None else base64.b64encode(raw).decode("ascii") + + +def _unb64(value): + if value is None: + return None + require(isinstance(value, str), "invalid_encoded_bytes") + try: + raw = base64.b64decode(value, validate=True) + except (ValueError, TypeError) as error: + raise ValueError("migration_invalid_encoded_bytes") from error + require(_b64(raw) == value and len(raw) <= admission.MAX_BYTES, "invalid_encoded_bytes") + return raw + + +def _canonical(path): + path = Path(path) + require(path.is_absolute() and str(path) == str(path.resolve(strict=True)), "noncanonical_path") + return path + + +def _directory(path): + path = _canonical(path) + info = path.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() + and not info.st_mode & 0o022, "unsafe_directory") + return info + + +def _identity(info): + return {"device": info.st_dev, "inode": info.st_ino, "uid": info.st_uid, + "gid": info.st_gid, "mode": info.st_mode, "size": info.st_size, + "mtime_ns": info.st_mtime_ns, "ctime_ns": info.st_ctime_ns, + "nlink": info.st_nlink} + + +def _read(path, *, absent=False, private=True, maximum=admission.MAX_BYTES): + """Descriptor-bound, bounded read with exact ownership and replacement checks.""" + path = Path(path) + _directory(path.parent) + try: + before = path.lstat() + except FileNotFoundError: + require(absent, "missing_file") + return None, None + require(stat.S_ISREG(before.st_mode) and before.st_uid == os.getuid() + and before.st_nlink == 1 and (not private or not before.st_mode & 0o077), "unsafe_file") + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + require(_identity(os.fstat(fd)) == _identity(before), "file_changed") + require(before.st_size <= maximum, "file_too_large") + chunks, remaining = [], maximum + 1 + while remaining: + data = os.read(fd, min(remaining, 65536)) + if not data: + break + chunks.append(data) + remaining -= len(data) + raw = b"".join(chunks) + require(len(raw) <= maximum and _identity(os.fstat(fd)) == _identity(before) + == _identity(path.lstat()), "file_changed") + return raw, _identity(before) + finally: + os.close(fd) + + +def _fsync(directory): + fd = os.open(directory, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _new(path, raw): + _directory(path.parent) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + try: + with os.fdopen(fd, "wb", closefd=False) as stream: + stream.write(raw) + stream.flush() + os.fsync(fd) + finally: + os.close(fd) + _fsync(path.parent) + + +def _mkdir(path): + _directory(path.parent) + try: + path.mkdir(mode=0o700) + _fsync(path.parent) + except FileExistsError: + pass + require(not _directory(path).st_mode & 0o077, "journal_directory_not_private") + + +class Locks: + """Never create or chmod app locks. Dry planning changes no application bytes.""" + def __init__(self, root): + self.paths = [root / "worker.lock", root.parent / "learner-provider.lock", root / "enqueue.lock"] + self.held = [] + + def __enter__(self): + try: + for path in self.paths: + _directory(path.parent) + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + info = os.fstat(fd) + require(stat.S_ISREG(info.st_mode) and info.st_uid == os.getuid() + and info.st_nlink == 1 and not info.st_mode & 0o077, "unsafe_lock") + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BaseException: + os.close(fd) + raise + self.held.append((path, fd, _identity(info))) + self.verify() + return self + except BaseException: + self.__exit__(None, None, None) + raise + + def verify(self): + for path, fd, expected in self.held: + require(_identity(path.lstat()) == _identity(os.fstat(fd)) == expected, "lock_changed") + + def __exit__(self, *_): + while self.held: + _, fd, _ = self.held.pop() + try: + fcntl.flock(fd, fcntl.LOCK_UN) + finally: + os.close(fd) + + +def _runtime(): + here = Path(__file__).resolve().parent + files = sorted(here.glob("*.py")) + [here.parent / "learner_router.py", here.parent / "codex_learner_migrate.py"] + require(1 <= len(files) <= 32, "runtime_membership_invalid") + return {str(path.relative_to(here.parent)): digest(_read(path, private=False)[0]) for path in files} + + +def _root(root): + root = _canonical(root) + require(not _directory(root).st_mode & 0o077, "state_directory_not_private") + for name in ("enrollments", "admissions", "sessions", "pending"): + require(not _directory(root / name).st_mode & 0o077, "state_directory_not_private") + return root + + +def _policy(value, root, stable): + require(set(value) == {"schema_version", "mode", "enabled", "activation_id", "cutoff", + "state_dir", "sessions_dir"}, "invalid_policy") + require(type(value["schema_version"]) is int and value["schema_version"] == (2 if stable else 1) + and value["mode"] == (MODE_V2 if stable else MODE_V1) + and type(value["enabled"]) is bool and value["state_dir"] == str(root), "invalid_policy") + admission._uuid(value["activation_id"], 4) + require(admission._timestamp(value["cutoff"]) <= datetime.now(timezone.utc), "future_cutoff") + if stable: + admission._bound_directory(value["sessions_dir"], stable_identity=True) + else: + _adopt_directory(value["sessions_dir"]) + + +def _adopt_directory(old): + require(isinstance(old, dict) and set(old) == {"path", "device", "inode"} + and all(type(old[k]) is int and old[k] > 0 for k in ("device", "inode")), "invalid_legacy_directory") + actual = admission._directory_binding(old["path"], stable_identity=True) + require(actual["identity"]["inode"] == old["inode"], "legacy_directory_inode_changed") + return actual + + +def _route(value, root, stable): + require(set(value) == {"schema_version", "mode", "enabled", "state_dir"} + and type(value["schema_version"]) is int and value["schema_version"] == (2 if stable else 1) + and value["mode"] == (MODE_V2 if stable else MODE_V1) + and type(value["enabled"]) is bool and value["state_dir"] == str(root), "invalid_route") + + +def _paths(root, sid): + admission._uuid(sid, 7) + return {name: root / name / (sid + ".json") for name in ("enrollments", "sessions", "admissions", "pending")} + + +def _binding(entry, policy, policy_raw): + return {**entry, "cutoff": policy["cutoff"], "policy_sha256": digest(policy_raw), + "state_dir": policy["state_dir"], "mode": policy["mode"]} + + +def _replace_identity(value, entry): + require(value.get("device") == entry["device"] and value.get("inode") == entry["inode"] + and "identity" not in value, "legacy_transcript_binding_invalid") + result = copy.deepcopy(value) + del result["device"], result["inode"] + return result + + +def _task_candidates(root, sid, before, legacy, legacy_raw, target, target_raw): + """Validate old chain and current anchors, changing only durable identity fields.""" + entry = admission._json(before["enrollments"]) + require(entry.get("session_id") == sid and type(entry.get("device")) is int + and type(entry.get("inode")) is int and entry["inode"] > 0 + and entry.get("uid") == os.getuid() and entry.get("activation_id") == legacy["activation_id"], + "legacy_enrollment_invalid") + payload = {"session_id": sid, "transcript_path": entry.get("transcript_path"), + "cwd": entry.get("project", {}).get("path"), "agent_id": sid} + origin = host_admission._origin(target, payload) + require(origin["identity"]["inode"] == entry["inode"], "legacy_transcript_inode_changed") + project = _adopt_directory(entry["project"]) + require(project == origin["project"], "project_identity_changed") + adopted = _replace_identity(entry, entry) + adopted.update(identity=origin["identity"], project=project) + require(set(adopted) == set(origin) | admission.FRONTIER_FIELDS | {"captured_at", "activation_id"} + and all(adopted[k] == value for k, value in origin.items()), "legacy_origin_changed") + captured = admission._timestamp(entry["captured_at"]) + require(admission._timestamp(legacy["cutoff"]) <= captured <= datetime.now(timezone.utc), "capture_time_invalid") + frontier = admission._frontier(origin, entry["frontier_offset"]) + require(all(entry[k] == value for k, value in frontier.items()), "legacy_frontier_changed") + old_binding = _binding(entry, legacy, legacy_raw) + new_binding = _binding(adopted, target, target_raw) + result = {"enrollments": encode(adopted), "sessions": None, "admissions": None, "pending": None} + record_raw, state_raw, request_raw = (before[n] for n in ("admissions", "sessions", "pending")) + if record_raw is None: + require(state_raw is None and request_raw is None, "orphan_state") + return result + require(state_raw is not None, "cursor_missing") + record, state = admission._json(record_raw), admission._json(state_raw) + require({"binding", "state_sha256", "processing", "last_event"} <= set(record) + and record["processing"] is None and record["binding"] == old_binding, "record_binding_or_processing_invalid") + require(record["state_sha256"] == object_digest(state) and state.get("admission") == old_binding + and state.get("session_id") == sid and state.get("transcript_path") == entry["transcript_path"] + and type(state.get("offset")) is int and state["offset"] >= entry["frontier_offset"], "cursor_chain_invalid") + require(state["offset"] <= Path(entry["transcript_path"]).stat().st_size, "cursor_truncated") + new_state = _replace_identity(state, entry) + new_state.update(identity=origin["identity"], admission=new_binding) + new_record = {**record, "binding": new_binding, "state_sha256": object_digest(new_state)} + require("legacy_event_identity" not in record, "legacy_event_identity_already_present") + if record["last_event"] is not None: + require(isinstance(record["last_event"], str) and re.fullmatch(r"[0-9a-f]{64}", record["last_event"]), "last_event_invalid") + new_record["legacy_event_identity"] = {"device": entry["device"], "inode": entry["inode"]} + result.update(sessions=encode(new_state), admissions=encode(new_record)) + if request_raw is not None: + request = admission._json(request_raw) + require(request.get("admission") == old_binding and request.get("session_id") == sid + and request.get("transcript_path") == entry["transcript_path"] + and request.get("cwd") == request.get("hook_cwd") == entry["project"]["path"] + and isinstance(request.get("request_id"), str) and bool(request["request_id"]) + and request.get("event") in host_admission.LEARN_EVENTS, "request_chain_invalid") + if request["event"] == "Stop": + require(isinstance(request.get("turn_id"), str) and re.fullmatch(r"[A-Za-z0-9_-]{1,100}", request["turn_id"]), "request_turn_invalid") + if request["event"] == "PreCompact": + require(request.get("trigger") == "auto", "request_trigger_invalid") + # Validate hook attribution through the maintained metadata parser. + hook_payload = {"session_id": request.get("hook_session_id"), "transcript_path": entry["transcript_path"], + "cwd": request["hook_cwd"], "agent_id": sid} + require(host_admission._origin(target, hook_payload) == origin, "request_origin_invalid") + new_request = _replace_identity(request, entry) + new_request.update(identity=origin["identity"], admission=new_binding) + result["pending"] = encode(new_request) + return result + + +def _record(path, before, info, after): + return {"path": str(path), "preimage": _b64(before), "preimage_sha256": None if before is None else digest(before), + "preimage_identity": info, "candidate": _b64(after), "candidate_sha256": None if after is None else digest(after)} + + +def _retained_legacy(root, path, legacy_raw, current_policy_raw): + """Later migrations must use the original bytes retained by a prior journal.""" + require(path.name == "legacy-policy.json" and path.parent.parent == root / "migrations" + and re.fullmatch(r"[0-9a-f]{64}", path.parent.name), "legacy_policy_not_retained_journal") + raw, _ = _read(path.parent / "plan.json", maximum=MAX_PLAN_BYTES) + require(digest(raw) == path.parent.name, "retained_plan_digest_changed") + old_plan = admission._json(raw) + require(raw == encode(old_plan) and old_plan.get("kind") == PLAN_KIND + and old_plan.get("root") == str(root) + and _unb64(old_plan["legacy_policy"]["bytes"]) == legacy_raw + and old_plan["legacy_policy"]["sha256"] == digest(legacy_raw) + and old_plan["records"][-1]["path"] == str(root / "admission.json") + and _unb64(old_plan["records"][-1]["candidate"]) == current_policy_raw, + "retained_policy_lineage_changed") + + +def _build(root, route_path, session_ids, legacy_policy_path): + policy_raw, policy_info = _read(root / "admission.json") + current = admission._json(policy_raw) + stable = current.get("mode") == MODE_V2 + _policy(current, root, stable) + route_raw, route_info = _read(route_path) + route = admission._json(route_raw) + _route(route, root, stable) + if legacy_policy_path is None: + require(not stable, "retained_legacy_policy_required") + legacy_raw, legacy_info = policy_raw, policy_info + legacy_path = root / "admission.json" + else: + legacy_path = _canonical(legacy_policy_path) + legacy_raw, legacy_info = _read(legacy_path) + legacy = admission._json(legacy_raw) + _policy(legacy, root, False) + require(all(current[k] == legacy[k] for k in ("activation_id", "cutoff", "state_dir", "enabled")), "foreign_legacy_policy") + target = {**legacy, "schema_version": 2, "mode": MODE_V2, + "sessions_dir": _adopt_directory(legacy["sessions_dir"])} + if stable: + require(current == target, "current_policy_lineage_changed") + _retained_legacy(root, legacy_path, legacy_raw, policy_raw) + target_raw = policy_raw # Preserve exact existing v2 hash for every later selection. + else: + require(legacy_raw == policy_raw, "legacy_policy_not_current") + target_raw = encode(target) + route_after = route_raw if stable else encode({**route, "schema_version": 2, "mode": MODE_V2}) + records, holds = [], [] + for sid in session_ids: + paths = _paths(root, sid) + before, infos = {}, {} + for name, path in paths.items(): + before[name], infos[name] = _read(path, absent=name != "enrollments") + candidate = _task_candidates(root, sid, before, legacy, legacy_raw, target, target_raw) + for name in paths: + records.append(_record(paths[name], before[name], infos[name], candidate[name])) + hold = root / "migration-holds" / (sid + ".json") + require(not os.path.lexists(hold), "existing_migration_hold") + holds.append(str(hold)) + records.append(_record(route_path, route_raw, route_info, route_after)) + records.append(_record(root / "admission.json", policy_raw, policy_info, target_raw)) + transition_raw = encode({**current, "enabled": False}) if current["enabled"] else policy_raw + return {"schema_version": 1, "kind": PLAN_KIND, "root": str(root), "route_path": str(route_path), + "session_ids": session_ids, "runtime_sha256": _runtime(), "records": records, "hold_paths": holds, + "transitional_policy": _record(root / "admission.json", policy_raw, policy_info, transition_raw), + "legacy_policy": {"path": str(legacy_path), "bytes": _b64(legacy_raw), "sha256": digest(legacy_raw), "identity": legacy_info}, + "historical_volume_continuity_proven": False, "owner_authorized_current_volume_adoption": True, + "learner_started": False, "publication_order": ORDER} + + +def prepare(root, route_path, session_ids, *, legacy_policy_path=None, authorize_current_volume_adoption=False): + require(authorize_current_volume_adoption is True, "explicit_identity_adoption_authorization_required") + root, route_path = _root(root), _canonical(route_path) + require(isinstance(session_ids, (list, tuple)) and 1 <= len(session_ids) <= MAX_SELECTED + and len(set(session_ids)) == len(session_ids), "invalid_selection") + selected = sorted(session_ids) + for sid in selected: + admission._uuid(sid, 7) + with Locks(root) as locks: + plan = _build(root, route_path, selected, legacy_policy_path) + require(len(encode(plan)) <= MAX_PLAN_BYTES, "plan_too_large") + require(plan == _build(root, route_path, selected, legacy_policy_path), "snapshot_changed") + locks.verify() + return plan + + +def _validate_plan(root, plan, expected_sha): + require(isinstance(plan, dict) and len(encode(plan)) <= MAX_PLAN_BYTES + and digest(encode(plan)) == expected_sha, "plan_sha_mismatch") + require(set(plan) == {"schema_version", "kind", "root", "route_path", "session_ids", "runtime_sha256", + "records", "hold_paths", "legacy_policy", "historical_volume_continuity_proven", + "owner_authorized_current_volume_adoption", "learner_started", "publication_order", "transitional_policy"} + and type(plan["schema_version"]) is int and plan["schema_version"] == 1 + and plan["kind"] == PLAN_KIND and plan["root"] == str(root) + and plan["publication_order"] == ORDER + and plan.get("owner_authorized_current_volume_adoption") is True + and plan.get("historical_volume_continuity_proven") is False and plan.get("learner_started") is False, + "invalid_plan") + selected = plan.get("session_ids") + require(isinstance(selected, list) and 1 <= len(selected) <= MAX_SELECTED + and selected == sorted(set(selected)), "invalid_selection") + expected_paths = [str(path) for sid in selected for path in _paths(root, sid).values()] + route_path = Path(plan["route_path"]) + require(route_path.is_absolute() and route_path not in [Path(p) for p in expected_paths] + and route_path != root / "admission.json", "invalid_route_path") + expected_paths += [str(route_path), str(root / "admission.json")] + require([r["path"] for r in plan["records"]] == expected_paths, "publication_paths_changed") + require(plan["hold_paths"] == [str(root / "migration-holds" / (sid + ".json")) for sid in selected], "hold_paths_changed") + require(plan["runtime_sha256"] == _runtime(), "runtime_changed") + for record in plan["records"]: + for raw_key, sha_key in (("preimage", "preimage_sha256"), ("candidate", "candidate_sha256")): + raw = _unb64(record[raw_key]) + require(record[sha_key] == (None if raw is None else digest(raw)), "record_digest_changed") + require((record["preimage"] is None) == (record["candidate"] is None) + == (record["preimage_identity"] is None), "record_absence_changed") + legacy = plan["legacy_policy"] + require(digest(_unb64(legacy["bytes"])) == legacy["sha256"], "legacy_policy_digest_changed") + legacy_path = Path(legacy["path"]) + if legacy_path != root / "admission.json": + raw, identity = _read(legacy_path) + require(raw == _unb64(legacy["bytes"]) and identity == legacy["identity"], "legacy_source_changed") + if admission._json(_unb64(plan["records"][-1]["preimage"]))["mode"] == MODE_V2: + _retained_legacy(root, legacy_path, raw, _unb64(plan["records"][-1]["preimage"])) + _validate_candidates(root, plan) + original_raw = _unb64(plan["records"][-1]["preimage"]) + original = admission._json(original_raw) + transition = encode({**original, "enabled": False}) if original["enabled"] else original_raw + require(plan["transitional_policy"] == _record(root / "admission.json", original_raw, + plan["records"][-1]["preimage_identity"], transition), "transitional_policy_changed") + + +def _validate_candidates(root, plan): + records = plan["records"] + original = admission._json(_unb64(records[-1]["preimage"])) + _policy(original, root, original.get("mode") == MODE_V2) + target_raw = _unb64(records[-1]["candidate"]) + target = admission._json(target_raw) + legacy_raw = _unb64(plan["legacy_policy"]["bytes"]) + legacy = admission._json(legacy_raw) + _policy(legacy, root, False) + _policy(target, root, True) + require(target == {**legacy, "schema_version": 2, "mode": MODE_V2, + "sessions_dir": _adopt_directory(legacy["sessions_dir"])}, "policy_transformation_invalid") + require(all(original[k] == legacy[k] for k in ("activation_id", "cutoff", "state_dir", "enabled")), "policy_lineage_invalid") + if original["mode"] == MODE_V1: + require(_unb64(records[-1]["preimage"]) == legacy_raw, "policy_lineage_invalid") + else: + require(records[-1]["preimage"] == records[-1]["candidate"], "stable_policy_must_not_change") + route_before = admission._json(_unb64(records[-2]["preimage"])) + route_after = admission._json(_unb64(records[-2]["candidate"])) + _route(route_before, root, original["mode"] == MODE_V2) + _route(route_after, root, True) + require(route_after == {**route_before, "schema_version": 2, "mode": MODE_V2}, "route_transformation_invalid") + for index, sid in enumerate(plan["session_ids"]): + batch = records[index * 4:index * 4 + 4] + before = {Path(r["path"]).parent.name: _unb64(r["preimage"]) for r in batch} + expected = _task_candidates(root, sid, before, legacy, legacy_raw, target, target_raw) + require(all(_unb64(r["candidate"]) == expected[Path(r["path"]).parent.name] for r in batch), "candidate_transformation_invalid") + + +def _hold(sid, plan_sha): + return encode({"schema_version": 1, "kind": HOLD_KIND, "session_id": sid, "plan_sha256": plan_sha}) + + +def _journal(root, plan, plan_sha, *, create): + parent, work = root / "migrations", root / "migrations" / plan_sha + if create: + _mkdir(parent) + _mkdir(work) + else: + require(not _directory(work).st_mode & 0o077, "journal_not_private") + files = {"plan.json": encode(plan), "legacy-policy.json": _unb64(plan["legacy_policy"]["bytes"])} + for name, raw in files.items(): + path = work / name + if create and not os.path.lexists(path): + _new(path, raw) + require(_read(path, maximum=MAX_PLAN_BYTES)[0] == raw, "journal_changed") + return work + + +def _event(work, label, **fields): + # Individual O_EXCL journal records avoid silently appending to a replaced log. + existing = list(work.glob("event-*.json")) + require(len(existing) < 10000, "journal_full") + _new(work / ("event-%05d.json" % (len(existing) + 1)), encode({"event": label, **fields})) + + +def _match(record, *, recover): + raw, info = _read(Path(record["path"]), absent=record["preimage"] is None) + before, after = _unb64(record["preimage"]), _unb64(record["candidate"]) + if raw == before and (before is None or info == record["preimage_identity"]): + return "before" + require(recover and raw == after and before is not None, "cas_mismatch") + return "after" + + +def _policy_phase(plan, *, recover): + final = plan["records"][-1] + raw, info = _read(Path(final["path"])) + original, transition, candidate = (_unb64(final["preimage"]), + _unb64(plan["transitional_policy"]["candidate"]), + _unb64(final["candidate"])) + if raw == original and info == final["preimage_identity"]: + # An already disabled original is the safe transition without a rewrite. + return "transition" if original == transition else "original" + require(recover, "cas_mismatch") + if raw == candidate: + return "final" + require(raw == transition, "policy_cas_mismatch") + return "transition" + + +def _verify_prefix(plan, *, recover): + states = [_match(r, recover=recover) for r in plan["records"][:-1]] + phase = _policy_phase(plan, recover=recover) + remaining = False + published = False + for state, record in zip(states, plan["records"][:-1]): + if record["preimage"] == record["candidate"]: + continue + if state == "before": + remaining = True + else: + require(not remaining, "nonprefix_publication") + published = True + require(not published or phase != "original", "records_published_without_transition") + require(phase != "final" or not remaining, "final_policy_published_before_records") + return states + ["after" if phase == "final" else "before"] + + +def _publish(record, work, index): + path = Path(record["path"]) + raw = _unb64(record["candidate"]) + _match(record, recover=False) + temporary = path.parent / (".identity-migration-" + work.name + "-" + str(index)) + if os.path.lexists(temporary): + require(_read(temporary)[0] == raw, "staging_file_changed") + else: + _new(temporary, raw) + _match(record, recover=False) + os.replace(temporary, path) + _fsync(path.parent) + require(_read(path)[0] == raw, "publication_changed") + + +def _publish_policy(plan, work, *, final): + """CAS the shared path against its precise current stage, never a rollback.""" + target = plan["records"][-1] if final else plan["transitional_policy"] + path = Path(target["path"]) + expected = _unb64(plan["transitional_policy"]["candidate"] if final else target["preimage"]) + raw, info = _read(path) + require(raw == expected, "policy_stage_changed") + after = _unb64(target["candidate"]) + if raw == after: + return + record = _record(path, raw, info, after) + _publish(record, work, "final-policy" if final else "transitional-policy") + + +def _run(root, plan, plan_sha, *, recovery): + root = _root(root) + with Locks(root) as locks: + _validate_plan(root, plan, plan_sha) + states = _verify_prefix(plan, recover=recovery) + work = _journal(root, plan, plan_sha, create=not recovery) + if _policy_phase(plan, recover=True) == "original": + locks.verify() + _event(work, "before_transitional_policy", candidate_sha256=plan["transitional_policy"]["candidate_sha256"]) + _publish_policy(plan, work, final=False) + _event(work, "transitional_policy_published", candidate_sha256=plan["transitional_policy"]["candidate_sha256"]) + _mkdir(root / "migration-holds") + # All holds must be durable before the first task record can change. + for sid, path in zip(plan["session_ids"], plan["hold_paths"]): + path, expected = Path(path), _hold(sid, plan_sha) + if os.path.lexists(path): + require(_read(path)[0] == expected, "hold_changed") + else: + require(all(s == "before" or r["preimage"] == r["candidate"] + for s, r in zip(states[:-1], plan["records"][:-1])), "hold_missing_after_publication") + _new(path, expected) + _event(work, "hold_published", session_id=sid) + for index, record in enumerate(plan["records"][:-1]): + locks.verify() + _validate_plan(root, plan, plan_sha) + _verify_prefix(plan, recover=True) + for sid, path in zip(plan["session_ids"], plan["hold_paths"]): + require(_read(Path(path))[0] == _hold(sid, plan_sha), "hold_changed") + if record["preimage"] == record["candidate"] or _match(record, recover=True) == "after": + continue + _event(work, "before_publish", index=index, path=record["path"], candidate_sha256=record["candidate_sha256"]) + _publish(record, work, index) + _event(work, "published", index=index, path=record["path"], candidate_sha256=record["candidate_sha256"]) + locks.verify() + _validate_plan(root, plan, plan_sha) + _verify_prefix(plan, recover=True) + if _policy_phase(plan, recover=True) != "final": + _event(work, "before_final_policy", candidate_sha256=plan["records"][-1]["candidate_sha256"]) + _publish_policy(plan, work, final=True) + _event(work, "final_policy_published", candidate_sha256=plan["records"][-1]["candidate_sha256"]) + _verify_complete(root, plan, plan_sha) + locks.verify() + _event(work, "complete", learner_started=False) + return {"status": "migration_complete_held", "plan_sha256": plan_sha, + "session_ids": plan["session_ids"], "learner_started": False, + "historical_volume_continuity_proven": False} + + +def apply(root, plan, plan_sha256): + return _run(root, plan, plan_sha256, recovery=False) + + +def recover(root, plan, plan_sha256): + return _run(root, plan, plan_sha256, recovery=True) + + +def _known_releases(root, work, plan, plan_sha): + result = set() + intents = list(work.glob("release-*.json")) + require(len(intents) <= MAX_SELECTED, "too_many_release_intents") + for path in intents: + raw, _ = _read(path) + value = admission._json(raw) + require(path.name == "release-" + digest(raw) + ".json" and raw == encode(value), "release_intent_changed") + _validate_release(root, plan, plan_sha, value) + result.update(value["session_ids"]) + return result + + +def _released_chain(root, plan, sid): + """Read-only validation of a previously released task's legitimate progress. + + Enrollment and policy remain the exact migrated anchor. Cursor, gate, pending + and processing metadata may advance after a separately authorized release. + No such file is rewritten by this command. All unreleased tasks still require + exact candidate bytes in _verify_complete. + """ + index = plan["session_ids"].index(sid) + records = plan["records"][index * 4:index * 4 + 4] + snapshots = {Path(r["path"]).parent.name: _read(Path(r["path"]), absent=True) for r in records} + if all(snapshots[Path(r["path"]).parent.name][0] == _unb64(r["candidate"]) for r in records): + # Includes deliberately disabled policies: exact already-validated chains + # need no public activation check to prove that no progression occurred. + return + entry_raw = snapshots["enrollments"][0] + require(entry_raw == _unb64(records[0]["candidate"]), "released_enrollment_changed") + entry = admission._json(entry_raw) + policy_raw = _unb64(plan["records"][-1]["candidate"]) + policy = admission._json(policy_raw) + payload = {"session_id": sid, "agent_id": sid, "transcript_path": entry["transcript_path"], + "cwd": entry["project"]["path"]} + require(host_admission._validate_entry(root, policy, payload) == entry, "released_origin_changed") + binding = _binding(entry, policy, policy_raw) + record_raw, state_raw, request_raw = (snapshots[k][0] for k in ("admissions", "sessions", "pending")) + if record_raw is None: + require(state_raw is None and request_raw is None and records[1]["candidate"] is None, + "released_cursor_missing") + else: + require(state_raw is not None, "released_cursor_missing") + record, state = admission._json(record_raw), admission._json(state_raw) + require({"binding", "state_sha256", "processing", "last_event"} <= set(record) + and record["binding"] == binding and record["state_sha256"] == object_digest(state) + and state.get("admission") == binding and state.get("session_id") == sid + and state.get("transcript_path") == entry["transcript_path"] + and state.get("identity") == entry["identity"] and "device" not in state and "inode" not in state, + "released_cursor_chain_invalid") + previous = admission._json(_unb64(records[1]["candidate"])) if records[1]["candidate"] is not None else None + floor = previous["offset"] if previous is not None else entry["frontier_offset"] + require(type(state.get("offset")) is int and floor <= state["offset"] <= Path(entry["transcript_path"]).stat().st_size, + "released_cursor_regressed_or_truncated") + if request_raw is not None: + request = admission._json(request_raw) + require(isinstance(request.get("request_id"), str) and bool(request["request_id"]) + and request.get("admission") == binding and request.get("session_id") == sid + and request.get("identity") == entry["identity"] and "device" not in request and "inode" not in request, + "released_request_chain_invalid") + require(host_admission.check(root, request) == binding, "released_request_admission_changed") + # Processing may be non-null after an interrupted but independently + # admitted later run. It is outside this release's mutation scope. + for record in records: + name = Path(record["path"]).parent.name + require(_read(Path(record["path"]), absent=True) == snapshots[name], "released_chain_changed_during_validation") + + +def _verify_complete(root, plan, plan_sha, *, released=()): + _validate_plan(root, plan, plan_sha) + work = _journal(root, plan, plan_sha, create=False) + authorized = _known_releases(root, work, plan, plan_sha) + require(set(released) <= authorized, "release_intent_missing") + absent = {sid for sid, path in zip(plan["session_ids"], plan["hold_paths"]) if not os.path.lexists(path)} + require(absent <= authorized, "hold_missing_without_release_intent") + for index, sid in enumerate(plan["session_ids"]): + if sid in absent: + _released_chain(root, plan, sid) + else: + for record in plan["records"][index * 4:index * 4 + 4]: + require(_read(Path(record["path"]), absent=record["candidate"] is None)[0] + == _unb64(record["candidate"]), "completed_chain_changed") + for record in plan["records"][-2:]: + require(_read(Path(record["path"]), absent=record["candidate"] is None)[0] == _unb64(record["candidate"]), "completed_chain_changed") + for sid, path in zip(plan["session_ids"], plan["hold_paths"]): + if not os.path.lexists(path): + require(sid in authorized, "hold_missing_without_release_intent") + else: + require(_read(Path(path))[0] == _hold(sid, plan_sha), "hold_changed") + # _validate_candidates re-runs maintained origin/frontier checks against the + # retained descriptors; exact postimages prove the chain without bypassing a + # normal host check's mandatory migration hold or disabled policy. + + +def prepare_release(root, plan, plan_sha256, session_ids): + root = _root(root) + selected = sorted(session_ids) + require(1 <= len(selected) <= MAX_SELECTED and len(set(selected)) == len(selected) + and set(selected) <= set(plan["session_ids"]), "invalid_release_selection") + with Locks(root) as locks: + _verify_complete(root, plan, plan_sha256) + records = [] + for sid in selected: + path = root / "migration-holds" / (sid + ".json") + raw, info = _read(path) + records.append({"path": str(path), "bytes": _b64(raw), "identity": info}) + locks.verify() + return {"schema_version": 1, "kind": "selected_identity_hold_release", "root": str(root), + "migration_plan_sha256": plan_sha256, "session_ids": selected, "holds": records, + "runtime_sha256": _runtime(), "learner_started": False} + + +def _validate_release(root, plan, plan_sha, release_plan): + require(set(release_plan) == {"schema_version", "kind", "root", "migration_plan_sha256", + "session_ids", "holds", "runtime_sha256", "learner_started"} + and type(release_plan["schema_version"]) is int and release_plan["schema_version"] == 1 + and release_plan["kind"] == "selected_identity_hold_release" + and release_plan["root"] == str(root) and release_plan["migration_plan_sha256"] == plan_sha + and release_plan["runtime_sha256"] == plan["runtime_sha256"] + and release_plan["learner_started"] is False, "invalid_release_plan") + selected = release_plan["session_ids"] + require(isinstance(selected, list) and selected == sorted(set(selected)) and 1 <= len(selected) <= MAX_SELECTED + and set(selected) <= set(plan["session_ids"]), "invalid_release_selection") + require([r["path"] for r in release_plan["holds"]] + == [str(root / "migration-holds" / (sid + ".json")) for sid in selected], "release_paths_changed") + for sid, record in zip(selected, release_plan["holds"]): + require(set(record) == {"path", "bytes", "identity"} + and _unb64(record["bytes"]) == _hold(sid, plan_sha), "release_hold_changed") + + +def release(root, plan, plan_sha256, release_plan, release_sha256): + root = _root(root) + require(digest(encode(release_plan)) == release_sha256, "release_sha_mismatch") + with Locks(root) as locks: + # An interrupted partial release requires the exact same release plan; + # the durable release intent below is the sole evidence allowing absence. + _validate_release(root, plan, plan_sha256, release_plan) + selected = release_plan["session_ids"] + work = _journal(root, plan, plan_sha256, create=False) + intent = work / ("release-" + release_sha256 + ".json") + absent = [sid for sid in selected if not os.path.lexists(root / "migration-holds" / (sid + ".json"))] + if os.path.lexists(intent): + require(_read(intent)[0] == encode(release_plan), "release_intent_changed") + else: + require(not absent, "hold_missing_without_release_intent") + _verify_complete(root, plan, plan_sha256) + for record in release_plan["holds"]: + raw, info = _read(Path(record["path"])) + require(raw == _unb64(record["bytes"]) and info == record["identity"], "release_hold_cas_mismatch") + _new(intent, encode(release_plan)) + for sid, record in zip(selected, release_plan["holds"]): + _verify_complete(root, plan, plan_sha256, released=absent) + locks.verify() + if sid in absent: + continue + path = Path(record["path"]) + raw, info = _read(path) + require(raw == _unb64(record["bytes"]) == _hold(sid, plan_sha256) + and info == record["identity"], "release_hold_cas_mismatch") + path.unlink() + _fsync(path.parent) + absent.append(sid) + _event(work, "hold_released", session_id=sid, release_sha256=release_sha256) + locks.verify() + return {"status": "selected_migration_holds_released", "session_ids": selected, "learner_started": False} + + +def _load_plan(path, expected): + raw, _ = _read(_canonical(path), maximum=MAX_PLAN_BYTES) + require(digest(raw) == expected, "plan_file_sha_mismatch") + plan = admission._json(raw) + require(raw == encode(plan), "noncanonical_plan_encoding") + return plan + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("plan", "apply", "recover", "release-plan", "release")) + parser.add_argument("--state-dir", required=True, type=Path) + parser.add_argument("--routes", type=Path) + parser.add_argument("--session-id", action="append", default=[]) + parser.add_argument("--legacy-policy", type=Path) + parser.add_argument("--authorize-current-volume-adoption", action="store_true") + parser.add_argument("--plan", type=Path) + parser.add_argument("--plan-sha256") + parser.add_argument("--release-plan", type=Path) + parser.add_argument("--release-sha256") + parser.add_argument("--output", type=Path) + args = parser.parse_args(argv) + def deadline(*_): + raise ValueError("migration_deadline_or_interruption") + handlers = {number: signal.signal(number, deadline) for number in (signal.SIGALRM, signal.SIGTERM, signal.SIGINT)} + signal.setitimer(signal.ITIMER_REAL, 30) + try: + if args.command == "plan": + require(args.routes is not None and args.output is not None, "plan_requires_routes_and_output") + result = prepare(args.state_dir, args.routes, args.session_id, legacy_policy_path=args.legacy_policy, + authorize_current_volume_adoption=args.authorize_current_volume_adoption) + else: + require(args.plan is not None and args.plan_sha256 is not None, "reviewed_plan_required") + plan = _load_plan(args.plan, args.plan_sha256) + if args.command == "release-plan": + require(args.output is not None, "release_plan_output_required") + result = prepare_release(args.state_dir, plan, args.plan_sha256, args.session_id) + elif args.command == "release": + require(args.release_plan is not None and args.release_sha256 is not None, "reviewed_release_required") + result = release(args.state_dir, plan, args.plan_sha256, + _load_plan(args.release_plan, args.release_sha256), args.release_sha256) + else: + result = (apply if args.command == "apply" else recover)(args.state_dir, plan, args.plan_sha256) + if args.output is not None: + _new(args.output, encode(result)) + print(json.dumps({"status": result.get("status", "review_required"), "sha256": digest(encode(result)), "learner_started": False})) + return 0 + except (OSError, ValueError, TypeError, KeyError) as error: + reason = str(error) if isinstance(error, ValueError) and re.fullmatch(r"(?:admission|migration)_[a-z_]+", str(error)) else "migration_failed" + print(json.dumps({"status": "refused_or_partial_recovery_required", "reason": reason, "learner_started": False})) + return 1 + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + for number, handler in handlers.items(): + signal.signal(number, handler) diff --git a/codex/plugins/engram/scripts/codex_learner/runner.py b/codex/plugins/engram/scripts/codex_learner/runner.py index 2c61c16..6a42a52 100644 --- a/codex/plugins/engram/scripts/codex_learner/runner.py +++ b/codex/plugins/engram/scripts/codex_learner/runner.py @@ -25,7 +25,7 @@ from .transcript import inspect_rollout, read_excerpt from .memory_proxy import verified_no_write_receipt -from . import admission, memory_config +from . import admission, memory_config, host_admission EVENTS = {"Stop", "SubagentStop", "PreCompact", "SessionEnd"} GUARD = "ENGRAM_CODEX_LEARNER" @@ -169,7 +169,10 @@ def validate_request(payload: dict, root: Path) -> dict: st = path.stat() if not path.is_file() or st.st_uid != os.getuid(): raise ValueError("transcript_not_owned_regular_file") - meta = inspect_rollout(str(path)) + stable = admission.check_activation(root).get("mode") == host_admission.MODE_V2 + meta = inspect_rollout(str(path), **({"stable_identity": True} if stable else {})) + if (st.st_dev, st.st_ino) != (meta.device, meta.inode): + raise ValueError("transcript_replaced_during_validation") agent_id = payload.get("agent_id") if agent_id is not None and agent_id != meta.session_id: raise ValueError("transcript_session_mismatch") @@ -196,6 +199,10 @@ def validate_request(payload: dict, root: Path) -> dict: "trigger": payload.get("trigger"), "requested_at": utc(), "size_bytes": st.st_size, "mtime_ns": st.st_mtime_ns, "device": st.st_dev, "inode": st.st_ino, "request_id": uuid.uuid4().hex} + if stable: + request.pop("device") + request.pop("inode") + request["identity"] = meta.identity request["admission"] = admission.check(root, request) return request @@ -205,6 +212,31 @@ def admission_digest(value: object) -> str: ensure_ascii=False, allow_nan=False).encode()).hexdigest() +def transcript_identity_fields(binding: dict) -> dict: + if binding.get("mode") == host_admission.MODE_V2: + return {"identity": binding["identity"]} + return {"device": binding["device"], "inode": binding["inode"]} + + +def event_digest(request: dict) -> str: + fields = {key: request.get(key) for key in ("event", "turn_id", "trigger", "size_bytes")} + fields.update(transcript_identity_fields({**request, "mode": request["admission"].get("mode")})) + return admission_digest(fields) + + +def legacy_event_matches(record: dict, request: dict) -> bool: + """One-way migration bridge using retained historical constants, never st_dev now.""" + if "legacy_event_identity" not in record: + return False + previous = record["legacy_event_identity"] + if (request["admission"].get("mode") != host_admission.MODE_V2 + or not isinstance(previous, dict) or set(previous) != {"device", "inode"} + or any(type(previous[key]) is not int or previous[key] < 0 for key in previous)): + raise ValueError("admission_invalid_legacy_event_identity") + fields = {key: request.get(key) for key in ("event", "turn_id", "trigger", "size_bytes")} + return record["last_event"] == admission_digest({**fields, **previous}) + + def admitted_record(root: Path, request: dict, *, create: bool = False) -> dict: """Called under enqueue.lock; never adopt a cursor or queue from another activation.""" binding = admission.check(root, request) @@ -219,7 +251,7 @@ def admitted_record(root: Path, request: dict, *, create: bool = False) -> dict: if state_path.exists() or paths["pending"].exists(): raise ValueError("admission_unowned_existing_state") state = {"session_id": sid, "transcript_path": binding["transcript_path"], - "device": binding["device"], "inode": binding["inode"], + **transcript_identity_fields(binding), "offset": binding["frontier_offset"], "admission": binding, "recent_messages": [], "current_turn_id": None, "status": "enrolled_frontier"} record = {"binding": binding, "state_sha256": admission_digest(state), @@ -241,6 +273,9 @@ def admitted_record(root: Path, request: dict, *, create: bool = False) -> dict: if (type(state.get("offset")) is not int or state["offset"] < binding["frontier_offset"] or state.get("admission") != binding): raise ValueError("admission_cursor_before_frontier") + if binding.get("mode") == host_admission.MODE_V2 and ( + state.get("identity") != binding["identity"] or "device" in state or "inode" in state): + raise ValueError("admission_cursor_binding_changed") return record @@ -257,7 +292,8 @@ def commit_admitted_state(root: Path, request: dict, state: dict) -> None: atomic_json(root / "sessions" / (sid + ".json"), state) -def enqueue(root: Path, payload: dict, *, spawn: bool = True, expected_admission: dict | None = None) -> bool: +def enqueue(root: Path, payload: dict, *, spawn: bool = True, expected_admission: dict | None = None, + route_guard=None) -> bool: if os.environ.get(GUARD) or os.environ.get("CLAUDE_MEMORY_LEARNER") or os.environ.get("ENGRAM_LEARNER_ORCHESTRATED") == "1": return False if payload.get("stop_hook_active"): @@ -274,6 +310,10 @@ def enqueue(root: Path, payload: dict, *, spawn: bool = True, expected_admission sid = request["session_id"] try: with lock_file(root / "enqueue.lock"): + # Trusted router callback, never obtained from hook input or config. + # Revalidate after waiting behind any policy/route migration. + if route_guard is not None: + route_guard() record = admitted_record(root, request, create=True) state = load_json(root / "sessions" / (sid + ".json")) if state.get("reconciliation_required"): @@ -281,14 +321,15 @@ def enqueue(root: Path, payload: dict, *, spawn: bool = True, expected_admission event_log(root, "reconciliation_required", session_id=sid, run_id=state.get("last_run"), reason=gate.get("reason"), memory_ids=gate.get("memory_ids", [])) return False - event_key = admission_digest({key: request.get(key) for key in - ("event", "turn_id", "trigger", "size_bytes", "device", "inode")}) - if record["last_event"] == event_key: + event_key = event_digest(request) + if record["last_event"] == event_key or legacy_event_matches(record, request): event_log(root, "duplicate_event", session_id=sid) return False # The durable receipt precedes the queue. A crash cannot mint a # second request for the same Stop or silently discard its cursor. - atomic_json(root / "admissions" / (sid + ".json"), {**record, "last_event": event_key}) + next_record = {**record, "last_event": event_key} + next_record.pop("legacy_event_identity", None) + atomic_json(root / "admissions" / (sid + ".json"), next_record) pending = root / "pending" / (sid + ".json") previous = load_pending(pending) # A final flush wins over a redundant Stop for the same snapshot. @@ -776,24 +817,30 @@ def process_request(root: Path, request: dict, config: dict, *, invoke=run_codex if state.get("retry_after", 0) > time.time(): return "backoff" path = Path(request["transcript_path"]) - meta = inspect_rollout(str(path)) + stable = request["admission"].get("mode") == host_admission.MODE_V2 + stable_kwargs = {"stable_identity": True} if stable else {} + meta = inspect_rollout(str(path), **stable_kwargs) if meta.session_id != sid: raise ValueError("transcript_session_changed") - if (request.get("device"), request.get("inode")) != (meta.device, meta.inode): + observed_identity = {"identity": meta.identity} if stable else {"device": meta.device, "inode": meta.inode} + if any(request.get(key) != value for key, value in observed_identity.items()): raise ValueError("transcript_replaced_since_enqueue") - if state and (state.get("device"), state.get("inode")) != (meta.device, meta.inode): + if state and any(state.get(key) != value for key, value in observed_identity.items()): raise ValueError("transcript_replaced") offset = state["offset"] # Durable admission requires a cursor at/after the sealed frontier. if meta.size_bytes < offset: raise ValueError("transcript_truncated") excerpt = read_excerpt(str(path), offset, max_chars=config["max_chars"], max_scan_bytes=config["max_scan_bytes"], - recent_messages=state.get("recent_messages"), current_turn_id=state.get("current_turn_id")) + recent_messages=state.get("recent_messages"), current_turn_id=state.get("current_turn_id"), + **stable_kwargs) if (excerpt.metadata.session_id, excerpt.metadata.device, excerpt.metadata.inode) != (sid, meta.device, meta.inode): raise ValueError("transcript_replaced_during_read") + if stable and excerpt.metadata.identity != meta.identity: + raise ValueError("transcript_replaced_during_read") next_state = {"session_id": sid, "hook_session_id": request.get("hook_session_id", sid), "transcript_path": str(path), "offset": excerpt.next_offset, "recent_messages": excerpt.recent_messages, "current_turn_id": excerpt.current_turn_id, - "device": meta.device, "inode": meta.inode, "updated_at": utc()} + **observed_identity, "updated_at": utc()} if excerpt.blocked_reason and not excerpt.text: event_log(root, "blocked", session_id=sid, reason=excerpt.blocked_reason, offset=offset) return "blocked" @@ -836,7 +883,7 @@ def process_request(root: Path, request: dict, config: dict, *, invoke=run_codex status=result["status"], writes=result.get("write_calls", 0)) if result["status"] != "succeeded": reconciliation = failure_reconciliation(run_dir, provider_started) - failure_state = {**state, "session_id": sid, "device": meta.device, "inode": meta.inode, + failure_state = {**state, "session_id": sid, **observed_identity, "status": "failed", "last_run": run_dir.name, "retry_after": time.time() + config["retry_seconds"]} if reconciliation is not None: diff --git a/codex/plugins/engram/scripts/codex_learner/runtime_identity.py b/codex/plugins/engram/scripts/codex_learner/runtime_identity.py index 484b068..feafb79 100644 --- a/codex/plugins/engram/scripts/codex_learner/runtime_identity.py +++ b/codex/plugins/engram/scripts/codex_learner/runtime_identity.py @@ -83,6 +83,9 @@ def capture(router_file, runner_module, admission_module, host_admission_module= "template": (template, ".md"), "admission": (admission_file, ".py")} if host_admission_module is not None: files["host_admission"] = (getattr(host_admission_module, "__file__", None), ".py") + file_identity_module = getattr(admission_module, "file_identity", None) + if file_identity_module is not None: + files["file_identity"] = (getattr(file_identity_module, "__file__", None), ".py") for name, (value, suffix) in files.items(): identity["sources"][name] = _observed_file(value, MAX_SOURCE_BYTES, suffix=suffix)[0] router_path = identity["sources"]["router"]["path"] diff --git a/codex/plugins/engram/scripts/codex_learner/transcript.py b/codex/plugins/engram/scripts/codex_learner/transcript.py index 71fdbe4..aeac1fb 100644 --- a/codex/plugins/engram/scripts/codex_learner/transcript.py +++ b/codex/plugins/engram/scripts/codex_learner/transcript.py @@ -13,12 +13,17 @@ from __future__ import annotations from dataclasses import dataclass +from contextlib import contextmanager import hashlib import json +import os from pathlib import Path import re +import stat as stat_module from typing import Any, BinaryIO +from . import file_identity + DEFAULT_MAX_CHARS = 24_000 DEFAULT_MAX_SCAN_BYTES = 8 * 1024 * 1024 @@ -47,6 +52,7 @@ class RolloutMetadata: device: int inode: int hook_session_id: str | None = None + identity: dict[str, Any] | None = None @dataclass(frozen=True) @@ -70,7 +76,32 @@ def _string(value: Any) -> str | None: return value if isinstance(value, str) and value else None -def inspect_rollout(path: str | Path) -> RolloutMetadata: +@contextmanager +def _rollout_stream(path: Path, *, stable_identity=False): + if not stable_identity: + with path.open("rb") as stream: + yield stream + return + if not path.is_absolute() or path.resolve(strict=True) != path: + raise TranscriptError("admission_noncanonical_path") + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK + with os.fdopen(os.open(path, flags), "rb") as stream: + before = os.fstat(stream.fileno()) + if not stat_module.S_ISREG(before.st_mode) or before.st_uid != os.getuid(): + raise TranscriptError("admission_file_not_owned_regular") + identity = file_identity.capture_fd(stream.fileno()) + yield stream + after = os.fstat(stream.fileno()) + current = path.stat(follow_symlinks=False) + def fields(value): + return value.st_dev, value.st_ino, value.st_uid, value.st_mode + if (path.resolve(strict=True) != path or fields(before) != fields(after) + or fields(current) != fields(after) + or file_identity.capture_fd(stream.fileno()) != identity): + raise TranscriptError("admission_transcript_identity_changed") + + +def inspect_rollout(path: str | Path, *, stable_identity=False) -> RolloutMetadata: """Inspect only the first complete metadata record, never its instructions. Subagents can put the logical root session's ID in ``session_id`` and their @@ -80,15 +111,20 @@ def inspect_rollout(path: str | Path) -> RolloutMetadata: override either identity. """ rollout = Path(path) - with rollout.open("rb") as stream: - return _inspect_stream(rollout, stream) - + with _rollout_stream(rollout, stable_identity=stable_identity) as stream: + return _inspect_stream(rollout, stream, stable_identity=stable_identity) -def _inspect_stream(path: Path, stream: BinaryIO) -> RolloutMetadata: - import os +def _inspect_stream(path: Path, stream: BinaryIO, *, stable_identity=False) -> RolloutMetadata: stat = os.fstat(stream.fileno()) + identity = file_identity.capture_fd(stream.fileno()) if stable_identity else None line = stream.readline(MAX_METADATA_BYTES + 1) + return _metadata_from_line(path, stat, line, stream.tell(), identity) + + +def _metadata_from_line(path: Path, stat: Any, line: bytes, start_offset: int, + identity: dict[str, Any] | None = None) -> RolloutMetadata: + """Parse metadata already read from the descriptor that supplied its identity.""" if len(line) > MAX_METADATA_BYTES or not line.endswith(b"\n"): raise TranscriptError("missing, incomplete, or oversized initial session metadata") try: @@ -142,12 +178,13 @@ def _inspect_stream(path: Path, stream: BinaryIO) -> RolloutMetadata: history_mode=_string(payload.get("history_mode")), history_start_ordinal=start_ordinal, fork_boundary_known=boundary_known, - start_offset=stream.tell(), + start_offset=start_offset, size_bytes=stat.st_size, mtime_ns=stat.st_mtime_ns, device=stat.st_dev, inode=stat.st_ino, hook_session_id=_string(payload.get("session_id")), + identity=identity, ) @@ -306,6 +343,7 @@ def read_excerpt( max_scan_bytes: int = DEFAULT_MAX_SCAN_BYTES, recent_messages: list[dict[str, Any]] | None = None, current_turn_id: str | None = None, + stable_identity: bool = False, ) -> Excerpt: """Return a bounded excerpt and the exact safely consumed byte boundary. @@ -328,8 +366,8 @@ def read_excerpt( chunks: list[str] = [] chars = 0 blocked = None - with Path(path).open("rb") as stream: - metadata = _inspect_stream(Path(path), stream) + with _rollout_stream(Path(path), stable_identity=stable_identity) as stream: + metadata = _inspect_stream(Path(path), stream, stable_identity=stable_identity) if start_offset > metadata.size_bytes: raise TranscriptError("cursor is beyond file size; rollout was truncated or replaced") if start_offset: diff --git a/codex/plugins/engram/scripts/codex_learner_migrate.py b/codex/plugins/engram/scripts/codex_learner_migrate.py new file mode 100644 index 0000000..badfc76 --- /dev/null +++ b/codex/plugins/engram/scripts/codex_learner_migrate.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +"""Guarded, explicit selected-task identity migration. See --help.""" +import sys + +sys.dont_write_bytecode = True + +from codex_learner.migration import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/codex/plugins/engram/scripts/engram_hook.py b/codex/plugins/engram/scripts/engram_hook.py index 8bddf96..3337d10 100644 --- a/codex/plugins/engram/scripts/engram_hook.py +++ b/codex/plugins/engram/scripts/engram_hook.py @@ -78,15 +78,26 @@ def initialize(codex_home: Path) -> Path: "relevant_project_recall": True, "tool_recall": True, "projects": {}, "selected_memory_ids": [], "wall_seconds": 2, "cooldown_seconds": 1, "context_chars": 6000}) + from codex_learner import admission, host_admission + policy_path = root / "learner" / "admission.json" + if not policy_path.exists() and not policy_path.is_symlink(): + initial_json(policy_path, { + "schema_version": 2, "mode": host_admission.MODE_V2, "enabled": True, + "activation_id": str(uuid.uuid4()), + "cutoff": datetime.now(timezone.utc).isoformat(), + "state_dir": str(root / "learner"), + "sessions_dir": admission._directory_binding(sessions, stable_identity=True)}) + else: + initial_json(policy_path, {}) # Validate existing ownership; never replace it. + raw, _ = admission._owned_bytes(policy_path, private=True) + current = admission._json(raw) + # Preserve an existing host's identity scheme, including disabled policy. + if (current.get("mode"), current.get("schema_version")) not in { + (host_admission.MODE, 1), (host_admission.MODE_V2, 2)}: + raise ValueError("unsupported_existing_host_policy") initial_json(root / "learner-routes.json", { - "schema_version": 1, "mode": "host_sessions_v1", "enabled": True, + "schema_version": current["schema_version"], "mode": current["mode"], "enabled": True, "state_dir": str(root / "learner")}) - initial_json(root / "learner" / "admission.json", { - "schema_version": 1, "mode": "host_sessions_v1", "enabled": True, - "activation_id": str(uuid.uuid4()), - "cutoff": datetime.now(timezone.utc).isoformat(), - "state_dir": str(root / "learner"), - "sessions_dir": {"path": str(sessions), "device": info.st_dev, "inode": info.st_ino}}) initial_json(root / "learner" / "settings.json", { "min_chars": 400, "max_chars": 12000, "max_scan_bytes": 8 * 1024 * 1024, "wall_seconds": 300, "max_tool_calls": 12, "max_writes": 3, diff --git a/codex/plugins/engram/scripts/learner_router.py b/codex/plugins/engram/scripts/learner_router.py index 5d3632f..045802a 100644 --- a/codex/plugins/engram/scripts/learner_router.py +++ b/codex/plugins/engram/scripts/learner_router.py @@ -41,9 +41,10 @@ def read_routes(path: Path) -> dict: raw, _ = admission._owned_bytes(path, private=True) require(len(raw) <= MAX_ROUTES_BYTES, "routes_too_large") policy = admission._json(raw) - if policy.get("mode") == host_admission.MODE: + if policy.get("mode") in host_admission.MODES: require(set(policy) == {"schema_version", "mode", "enabled", "state_dir"} - and type(policy["schema_version"]) is int and policy["schema_version"] == 1, + and type(policy["schema_version"]) is int + and policy["schema_version"] == (2 if policy["mode"] == host_admission.MODE_V2 else 1), "routes_schema_invalid") require(policy["enabled"] is True, "routes_inactive") root = admission._canonical(policy["state_dir"]) @@ -106,7 +107,7 @@ def dispatch(path: Path, payload: dict, *, spawn: bool = True) -> bool: phase, sid = "routes", None try: policy = read_routes(path) - if policy.get("mode") == host_admission.MODE: + if policy.get("mode") in host_admission.MODES: return dispatch_host(path, policy, payload, spawn=spawn) phase = "selection" require(isinstance(payload, dict) and payload.get("hook_event_name") == "Stop", "event_not_stop") @@ -157,6 +158,12 @@ def dispatch_host(path: Path, policy: dict, payload: dict, *, spawn=True) -> boo if event in {"SubagentStart", "SubagentStop"} and payload.get("agent_transcript_path"): payload["transcript_path"] = payload["agent_transcript_path"] root = admission._canonical(policy["state_dir"]) + def route_guard(): + require(read_routes(path) == policy, "admission_route_mismatch") + current, _ = host_admission.policy(root) + require((policy["mode"], policy["schema_version"]) == + (current["mode"], current["schema_version"]), "admission_route_mismatch") + route_guard() if event == "SessionEnd": # Claude parity: cleanup belongs to lifecycle adapter, not a new learner. receipt(path, {"phase": "cleanup", "status": "observed", "reason": "session_end_cleanup_only", @@ -164,6 +171,7 @@ def dispatch_host(path: Path, policy: dict, payload: dict, *, spawn=True) -> boo return False phase = "enrollment" with runner.lock_file(root / "enqueue.lock"): + route_guard() sid, enrolled, fresh_child = host_admission.observe(root, payload) if (enrolled and not fresh_child) or event not in host_admission.LEARN_EVENTS: receipt(path, {"phase": phase, "status": "enrolled" if enrolled else "observed", @@ -171,7 +179,7 @@ def dispatch_host(path: Path, policy: dict, payload: dict, *, spawn=True) -> boo "session_id": sid, "hook_event": event}) return False phase = "enqueue" - queued = runner.enqueue(root, payload, spawn=spawn) + queued = runner.enqueue(root, payload, spawn=spawn, route_guard=route_guard) receipt(path, {"phase": phase, "status": "queued" if queued else "not_queued", "reason": "queued" if queued else "enqueue_refused", "session_id": sid, "hook_event": event, "trigger": payload.get("trigger")}) diff --git a/codex/plugins/engram/skills/hook-status/SKILL.md b/codex/plugins/engram/skills/hook-status/SKILL.md index 9305f33..12f3ed6 100644 --- a/codex/plugins/engram/skills/hook-status/SKILL.md +++ b/codex/plugins/engram/skills/hook-status/SKILL.md @@ -13,6 +13,8 @@ Summarize advice, tool/lifecycle events, routing, policy, and learner completion `reconciliation_required` means a failed run may already have written memory, or its write status is unknown. Report its linked run, reason, and recorded UUIDs; later events are held to prevent blind replay. The status reader does not clear this gate or establish whether an uncertain write persisted. +`migration_hold` is a separate filesystem-identity migration hold. Report it independently of a reconciliation gate and a paused pending request. `host_sessions_v2` uses persistent APFS volume UUID and inode identities; legacy v1 enrollments remain held until explicitly migrated. A migration hold or `admission_legacy_migration_required` does not authorize a retry, clearing a gate, changing the cursor, or adopting the current end of the transcript. Migration and activation are separately reviewed operations. + Report a returned learner `reason` code when present. The reader exposes only an exact allowlist of source-defined admission codes; it omits other reasons rather than displaying arbitrary error text. A missing reason therefore does not mean the event had no error. Advice diagnostics and lifecycle failure kinds are exact enums, never exception text. Runtime identity exposes only package version, source hashes, and typed status; it identifies observed package files, not successful execution or saved memory. diff --git a/codex/plugins/engram/skills/hook-status/scripts/hook_status.py b/codex/plugins/engram/skills/hook-status/scripts/hook_status.py index e92ff26..dc6d9a6 100644 --- a/codex/plugins/engram/skills/hook-status/scripts/hook_status.py +++ b/codex/plugins/engram/skills/hook-status/scripts/hook_status.py @@ -64,7 +64,15 @@ "session_or_project_mismatch", "state_directory_not_private", "state_directory_symlink", "state_file_symlink", "transcript_outside_sessions", "unknown_inherited_boundary", "unowned_existing_state", "unqualified_event", "unqualified_source", "unqualified_version", - "unsupported_policy"}} + "unsupported_policy", "legacy_migration_required", "migration_held", + "directory_identity_changed", "invalid_transcript_binding", "invalid_initial_metadata_size", + "initial_metadata_changed", "invalid_legacy_event_identity"}} +REASONS |= {"admission_identity_" + value for value in { + "abi_unsupported", "api_unavailable", "attributes_malformed", "attributes_unavailable", + "attributes_unsupported", "fd_invalid", "file_type_unsupported", "filesystem_changed", + "filesystem_malformed", "filesystem_unavailable", "filesystem_unsupported", "inode_invalid", + "invalid", "mount_invalid", "persistence_unsupported", "platform_unsupported", "root_changed", + "root_mismatch", "scheme_unsupported", "target_changed", "unavailable", "uuid_invalid", "volume_changed"}} FAILURE_PHASES = {"arguments", "config", "state_setup", "input", "payload", "policy", "state_lock", "state_load", "state_reserve", "spawn", "mcp_initialize", "mcp_recall", "mcp_graph", "parse_recall", "parse_graph", "render", @@ -259,7 +267,7 @@ def observed_file(record): sources = value.get("sources") if isinstance(sources, dict): out["sources"] = {key: observed_file(sources[key]) for key in - ("router", "runner", "template", "admission", "host_admission") + ("router", "runner", "template", "admission", "host_admission", "file_identity") if isinstance(sources.get(key), dict)} return out @@ -290,6 +298,18 @@ def metadata(record): for key in ("status", "pause_reason"): if isinstance(record.get(key), str) and record[key] in STATUSES: out[key] = record[key] + stable = record.get("identity") + if (isinstance(stable, dict) and stable.get("scheme") == "macos_volume_uuid_inode_v1" + and valid_uuid(stable.get("volume_uuid")) + and type(stable.get("inode")) is int and 0 < stable["inode"] < 2 ** 64): + out["identity"] = {"scheme": stable["scheme"], "volume_uuid": stable["volume_uuid"], + "inode": stable["inode"]} + if record.get("mode") in {"host_sessions_v1", "host_sessions_v2"}: + out["mode"] = record["mode"] + if record.get("kind") == "identity_migration_hold": + out["kind"] = record["kind"] + if isinstance(record.get("plan_sha256"), str) and re.fullmatch(r"[0-9a-f]{64}", record["plan_sha256"]): + out["plan_sha256"] = record["plan_sha256"] if isinstance(record.get("reason"), str) and record["reason"] in REASONS: out["reason"] = record["reason"] for key, allowed in (("failure_phase", FAILURE_PHASES), ("error_kind", ERROR_KINDS)): @@ -346,10 +366,10 @@ def policy_view(path): if value is not None: out["metadata"] = {k: v for k, v in metadata(value).items() if k in {"enabled", "activation_id", "cutoff"}} - if value.get("mode") == "host_sessions_v1": - out["metadata"]["mode"] = "host_sessions_v1" - if type(value.get("schema_version")) is int and value["schema_version"] == 1: - out["metadata"]["schema_version"] = 1 + if value.get("mode") in {"host_sessions_v1", "host_sessions_v2"}: + out["metadata"]["mode"] = value["mode"] + if type(value.get("schema_version")) is int and value["schema_version"] in {1, 2}: + out["metadata"]["schema_version"] = value["schema_version"] return out @@ -400,6 +420,7 @@ def report(session_id=None, environ=None): _, admission_view = scoped_json(learner / "admissions" / (sid + ".json"), sid, binding=True) _, pending_view = scoped_json(learner / "pending" / (sid + ".json"), sid) _, enrollment_view = scoped_json(learner / "enrollments" / (sid + ".json"), sid) + _, migration_view = scoped_json(learner / "migration-holds" / (sid + ".json"), sid) out = {"status": "ok", "session_id": sid, "session_sha256_prefix": session_hash, "state_root": selection, "advice": {"window": advice_window, "receipts": [metadata(v) for v in advice], "delivery_evidence": "receipt_only_not_model_delivery"}, @@ -411,7 +432,7 @@ def report(session_id=None, environ=None): "policy": policy_view(learner / "admission.json"), "events": [metadata(v) for v in events], "session": state_view, "admission": admission_view, "enrollment": enrollment_view, - "pending": pending_view, + "pending": pending_view, "migration_hold": migration_view, "memory_readback": "not_performed"}} # A run must be named by a matching event or this exact session's state. linked = next((v.get("run_id") for v in reversed(events) if "run_id" in v), None) diff --git a/scripts/package_codex_plugin.py b/scripts/package_codex_plugin.py index 891a368..abf91cc 100644 --- a/scripts/package_codex_plugin.py +++ b/scripts/package_codex_plugin.py @@ -54,6 +54,7 @@ "codex_learner/memory_proxy.py", "codex_learner/memory_config.py", "codex_learner/runtime_identity.py", "codex_learner/stdio_bridge.py", "codex_learner/transcript.py", + "codex_learner/file_identity.py", "codex_learner/migration.py", "codex_learner_migrate.py", ) VERSION = re.compile(r"(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:alpha|beta|rc)\.[1-9]\d*)?") PRIVATE_PATH = re.compile(r"(?:/Users/|/home/)[^\s\"'<>/]+/|[A-Za-z]:\\{1,2}Users\\{1,2}", re.I) From 6d2eafaeb093d4aa6838e510d1fceb3f02032506 Mon Sep 17 00:00:00 2001 From: jsflax Date: Mon, 21 Sep 2026 14:11:36 -0400 Subject: [PATCH 3/8] Recheck healthy task impact before learner policy migration --- Tests/codex_plugin/test_migration.py | 226 +++++++++++++++++- codex/plugins/engram/README.md | 7 + .../engram/scripts/codex_learner/migration.py | 183 +++++++++++++- 3 files changed, 412 insertions(+), 4 deletions(-) diff --git a/Tests/codex_plugin/test_migration.py b/Tests/codex_plugin/test_migration.py index a3370f8..7aa30df 100644 --- a/Tests/codex_plugin/test_migration.py +++ b/Tests/codex_plugin/test_migration.py @@ -606,7 +606,7 @@ def _legacy_admission(self): return importlib.import_module(name + ".admission") def test_transitional_policy_refuses_actual_retained_v1_and_current_v2_workers(self): - unselected = self.fixture.task(2, paused=False, gated=False) + unselected = self.fixture.task(2, paused=False, gated=True) plan = self.prepare() legacy = self._legacy_admission() self.assertEqual(legacy.check_activation(self.root)["mode"], "host_sessions_v1") @@ -666,5 +666,229 @@ def test_early_final_policy_or_unknown_transition_is_refused(self): M.recover(self.root, plan, M.digest(M.encode(plan))) +class ImpactTests(unittest.TestCase): + """Only the new first-flip cohort assertion; old suite is run by the owner.""" + setUp = MigrationTests.setUp + prepare = MigrationTests.prepare + apply = MigrationTests.apply + snapshot = MigrationTests.snapshot + + def healthy(self, sid, *, processing=None): + paths = M._paths(self.root, sid) + state = json.loads(paths["sessions"].read_bytes()) + state.pop("reconciliation_required", None) + F.write_json(paths["sessions"], state) + record = json.loads(paths["admissions"].read_bytes()) + record.update(state_sha256=M.object_digest(state), processing=processing) + F.write_json(paths["admissions"], record) + + def unchanged_refusal(self, plan, reason="impact_healthy_cohort_changed"): + before = self.snapshot() + with self.assertRaisesRegex(ValueError, reason): + self.apply(plan) + self.assertEqual(before, self.snapshot()) + self.assertFalse((self.root / "migrations").exists()) + self.assertFalse((self.root / "migration-holds").exists()) + + def test_first_flip_rejects_omitted_healthy_cursor_without_pending(self): + self.fixture.task(2, pending=False, gated=False) + before = self.snapshot() + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_not_selected"): + self.prepare() + self.assertEqual(before, self.snapshot()) + + def test_first_flip_rejects_omitted_enrollment_only(self): + self.fixture.task(2, cursor=False, pending=False) + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_not_selected"): + self.prepare() + + def test_unchanged_selected_healthy_cohort_and_normal_paused_queue_pass(self): + paused = self.fixture.task(2, paused=True, gated=False) + no_cursor = self.fixture.task(3, cursor=False, pending=False) + plan = self.prepare([self.sid, paused, no_cursor]) + self.assertEqual(plan["activation_impact"]["healthy_session_ids"], sorted([paused, no_cursor])) + self.assertEqual(plan["activation_impact"]["max_tasks"], 500) + result = self.apply(plan) + self.assertEqual(result["status"], "migration_complete_held") + sha = M.digest(M.encode(plan)) + seal = self.root / "migrations" / sha / "activation-impact.json" + self.assertEqual(seal.read_bytes(), M._impact_seal(plan, sha)) + pending = json.loads(M._paths(self.root, paused)["pending"].read_bytes()) + self.assertEqual(pending["paused_request_id"], pending["request_id"]) + + def test_new_healthy_enrollment_after_plan_refuses_with_zero_mutation(self): + plan = self.prepare() + self.fixture.task(2, cursor=False, pending=False) + self.unchanged_refusal(plan) + + def test_existing_gate_cleared_after_plan_refuses_with_zero_mutation(self): + other = self.fixture.task(2, gated=True) + plan = self.prepare() + self.healthy(other) + self.unchanged_refusal(plan) + + def test_processing_completion_after_plan_refuses_with_zero_mutation(self): + other = self.fixture.task(2, gated=False) + self.healthy(other, processing="existing-worker-request") + plan = self.prepare() + self.healthy(other, processing=None) + self.unchanged_refusal(plan) + + def test_same_membership_origin_failure_becoming_valid_is_rechecked(self): + other = self.fixture.task(2, gated=False) + path = M._paths(self.root, other)["enrollments"] + original = path.read_bytes() + value = json.loads(original) + value["device"] += 99 + F.write_json(path, value) + plan = self.prepare() + path.write_bytes(original) + self.unchanged_refusal(plan) + + def test_runtime_admissible_unusual_cursor_is_counted_not_hidden_by_conversion(self): + other = self.fixture.task(2, gated=False) + paths = M._paths(self.root, other) + state = json.loads(paths["sessions"].read_bytes()) + for key in ("session_id", "transcript_path", "device", "inode"): + del state[key] + F.write_json(paths["sessions"], state) + record = json.loads(paths["admissions"].read_bytes()) + record["state_sha256"] = M.object_digest(state) + F.write_json(paths["admissions"], record) + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_not_selected"): + self.prepare() + with self.assertRaisesRegex(ValueError, "cursor_chain_invalid"): + self.prepare([self.sid, other]) + + def test_damaged_pending_does_not_hide_future_event_healthy_cursor(self): + other = self.fixture.task(2, gated=False) + path = M._paths(self.root, other)["pending"] + for raw in (b"not json\n", b'{"admission":"stale","request_id":null}\n'): + with self.subTest(raw=raw): + path.write_bytes(raw) + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_not_selected"): + self.prepare() + with self.assertRaises(ValueError): + self.prepare([self.sid, other]) + + def test_ignored_invalid_pending_does_not_mask_gate_clear_after_plan(self): + other = self.fixture.task(2, gated=True) + path = M._paths(self.root, other)["pending"] + path.write_bytes(b"not json\n") + plan = self.prepare() + self.healthy(other) + self.unchanged_refusal(plan) + + def test_runner_accepted_duplicate_cursor_json_is_counted_then_conversion_refuses(self): + other = self.fixture.task(2, gated=False) + paths = M._paths(self.root, other) + state = json.loads(paths["sessions"].read_bytes()) + normal = json.dumps(state) + self.assertEqual(normal[0], "{") + paths["sessions"].write_text('{"offset":0,' + normal[1:]) + # Runtime JSON decoding yields the same final object and valid digest. + self.assertEqual(json.loads(paths["sessions"].read_bytes()), state) + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_not_selected"): + self.prepare() + with self.assertRaisesRegex(ValueError, "admission_invalid_json"): + self.prepare([self.sid, other]) + + def test_existing_hold_cannot_hide_retained_v1_healthy_task(self): + other = self.fixture.task(2, gated=False) + parent = F.private_directory(self.root / "migration-holds") + F.write_json(parent / (other + ".json"), {"existing": "must not be replaced"}) + before = self.snapshot() + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_not_selected"): + self.prepare() + with self.assertRaisesRegex(ValueError, "existing_migration_hold"): + self.prepare([self.sid, other]) + self.assertEqual(before, self.snapshot()) + + def test_healthy_cohort_over_32_refuses_without_expansion_or_mutation(self): + for n in range(2, 35): + self.fixture.task(n, cursor=False, pending=False) + before = self.snapshot() + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_limit_exceeded"): + self.prepare() + self.assertEqual(before, self.snapshot()) + + def test_inventory_bound_counts_non_json_entries_too(self): + (self.root / "enrollments" / "junk-a").touch() + (self.root / "enrollments" / "junk-b").touch() + with mock.patch.object(M, "MAX_IMPACT_TASKS", 2): + with self.assertRaisesRegex(ValueError, "impact_inventory_limit_exceeded"): + self.prepare() + + def test_disabled_policy_keeps_latent_healthy_cohort_and_seals_initial_apply(self): + self.healthy(self.sid) + policy = {**self.fixture.policy, "enabled": False} + F.write_json(self.root / "admission.json", policy) + MigrationTests._repair_fixture_chain(self) + plan = self.prepare() + self.assertEqual(plan["activation_impact"]["healthy_session_ids"], [self.sid]) + self.assertIs(plan["activation_impact"]["policy_enabled"], False) + self.apply(plan) + sha = M.digest(M.encode(plan)) + self.assertTrue((self.root / "migrations" / sha / "activation-impact.json").exists()) + self.assertIs(json.loads((self.root / "admission.json").read_bytes())["enabled"], False) + + def test_recovery_after_transition_uses_seal_not_partial_record_reclassification(self): + other = self.fixture.task(2, gated=False) + plan = self.prepare([self.sid, other]) + sha = M.digest(M.encode(plan)) + original = M._publish + def stop(record, work, index): + original(record, work, index) + if index == 0: raise RuntimeError("selected enrollment converted") + with mock.patch.object(M, "_publish", side_effect=stop): + with self.assertRaises(RuntimeError): self.apply(plan) + with mock.patch.object(M, "_activation_impact", side_effect=AssertionError("must not reclassify partial records")): + M.recover(self.root, plan, sha) + + def test_missing_seal_after_transition_refuses_recovery(self): + plan = self.prepare() + sha = M.digest(M.encode(plan)) + original = M._event + def stop(work, label, **fields): + if label == "transitional_policy_published": raise RuntimeError("suspended") + original(work, label, **fields) + with mock.patch.object(M, "_event", side_effect=stop): + with self.assertRaises(RuntimeError): self.apply(plan) + seal = self.root / "migrations" / sha / "activation-impact.json" + seal.unlink() + before = self.snapshot() + with self.assertRaisesRegex(ValueError, "impact_seal_missing_after_transition"): + M.recover(self.root, plan, sha) + self.assertEqual(before, self.snapshot()) + + def test_recovery_before_transition_rechecks_newly_healthy_task(self): + plan = self.prepare() + sha = M.digest(M.encode(plan)) + original = M._event + def stop(work, label, **fields): + if label == "before_transitional_policy": raise RuntimeError("not suspended yet") + original(work, label, **fields) + with mock.patch.object(M, "_event", side_effect=stop): + with self.assertRaises(RuntimeError): self.apply(plan) + self.fixture.task(2, cursor=False, pending=False) + before = self.snapshot() + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_changed"): + M.recover(self.root, plan, sha) + self.assertEqual(before, self.snapshot()) + + def test_later_v2_selection_keeps_policy_hash_without_v1_rescan(self): + other = self.fixture.task(2, gated=True) + first = self.prepare() + self.apply(first) + sha = M.digest(M.encode(first)) + legacy = self.root / "migrations" / sha / "legacy-policy.json" + policy_raw = (self.root / "admission.json").read_bytes() + with mock.patch.object(M, "_activation_impact", side_effect=AssertionError("no v1 rescan for later v2 migration")): + later = self.prepare([other], legacy_policy_path=legacy) + self.assertIsNone(later["activation_impact"]) + self.apply(later) + self.assertEqual((self.root / "admission.json").read_bytes(), policy_raw) + + if __name__ == "__main__": unittest.main() diff --git a/codex/plugins/engram/README.md b/codex/plugins/engram/README.md index 4aeedfd..dbbb30e 100644 --- a/codex/plugins/engram/README.md +++ b/codex/plugins/engram/README.md @@ -96,6 +96,13 @@ before passing its path and SHA256 to `apply`. When no historical volume UUID exists, current-volume adoption must be explicitly authorized and is recorded as such; migration does not prove historical volume continuity. +The first host policy conversion must include the explicitly reviewed healthy +v1 tasks, including tasks waiting for their next event. The tool rechecks that +cohort under its locks before changing state. A newly eligible task causes a +refusal and a new review; it is never silently added. If the required cohort +exceeds the supported selection bound, keep v1 until a coordinated rollout is +prepared. Installing the new runtime preserves an existing v1 policy. + Applying a plan temporarily disables host admission while publishing the selected records and route, then restores the policy's original enabled setting in v2. The journal supports `recover` after an interrupted publication. Other tasks' diff --git a/codex/plugins/engram/scripts/codex_learner/migration.py b/codex/plugins/engram/scripts/codex_learner/migration.py index 7882bdd..e2a3e91 100644 --- a/codex/plugins/engram/scripts/codex_learner/migration.py +++ b/codex/plugins/engram/scripts/codex_learner/migration.py @@ -24,6 +24,7 @@ from . import admission, file_identity, host_admission MAX_SELECTED = 32 +MAX_IMPACT_TASKS = 500 MAX_PLAN_BYTES = 32 * 1024 * 1024 MODE_V1 = "host_sessions_v1" MODE_V2 = "host_sessions_v2" @@ -336,6 +337,147 @@ def _retained_legacy(root, path, legacy_raw, current_policy_raw): "retained_policy_lineage_changed") +def _impact_members(root): + """Bound every directory entry, including malformed/non-JSON names.""" + parent = root / "enrollments" + _directory(parent) + result = [] + with os.scandir(parent) as entries: + for index, entry in enumerate(entries): + require(index < MAX_IMPACT_TASKS, "impact_inventory_limit_exceeded") + if not entry.name.endswith(".json"): + continue + sid = entry.name[:-5] + try: + admission._uuid(sid, 7) + except ValueError: + continue # Exactly as maintained host enrollment enumeration. + result.append(sid) + return sorted(result) + + +def _impact_task(root, policy, policy_raw, sid): + paths = _paths(root, sid) + # Unsafe/unbounded metadata cannot be evidence that a task is not healthy. + # Refuse the complete assertion instead of silently classifying it as held. + snapshots = {name: _read(path, absent=name != "enrollments") for name, path in paths.items()} + healthy = False + try: + # Retained v1 runtimes predate migration holds. A marker must not hide a + # task those runtimes could still admit; selected conversion separately + # refuses any existing hold instead of overwriting or releasing it. + entry = admission._json(snapshots["enrollments"][0]) + admission._require(entry.get("session_id") == sid and "identity" not in entry, "enrollment_origin_changed") + payload = {"session_id": sid, "agent_id": sid, "transcript_path": entry.get("transcript_path"), + "cwd": entry.get("project", {}).get("path")} + admission._require(host_admission._validate_entry(root, policy, payload) == entry, "enrollment_origin_changed") + binding = _binding(entry, policy, policy_raw) + record_raw, state_raw, pending_raw = (snapshots[name][0] for name in ("admissions", "sessions", "pending")) + if record_raw is None and state_raw is None and pending_raw is None: + healthy = True + else: + admission._require(record_raw is not None and state_raw is not None, "missing_durable_record") + # runner.load_json uses ordinary JSON decoding for these two files. + # Preserve its last-key-wins semantics in the coverage predicate; + # strict conversion can subsequently refuse duplicate-key records. + try: + record, state = (json.loads(raw.decode("utf-8")) for raw in (record_raw, state_raw)) + except (UnicodeError, ValueError, RecursionError) as error: + raise ValueError("admission_invalid_json") from error + admission._require(isinstance(record, dict) and isinstance(state, dict), "json_not_object") + # Match v1 runner.admitted_record(create=False), NOT the deliberately + # stricter conversion predicates. An admissible unusual record must + # be selected or refuse conversion, never disappear from impact. + admission._require(record.get("binding") == binding, "durable_binding_changed") + admission._require({"state_sha256", "last_event", "processing"} <= set(record), "incomplete_durable_record") + admission._require(record["state_sha256"] is not None + and record["state_sha256"] == object_digest(state), "cursor_missing_or_changed") + admission._require(type(state.get("offset")) is int and state["offset"] >= binding["frontier_offset"] + and state.get("admission") == binding, "cursor_before_frontier") + # A genuinely new enqueue can replace a damaged/stale pending item. + # Future-event coverage therefore depends on the admitted cursor, + # not pending validity or pause. Conversion remains independently + # strict and may refuse such a selected task until it is repaired. + healthy = not bool(state.get("reconciliation_required")) and record["processing"] is None + admission._require(host_admission._validate_entry(root, policy, payload) == entry, "enrollment_origin_changed") + except ValueError as error: + if not str(error).startswith("admission_"): + raise + healthy = False + except (OSError, KeyError, TypeError, RuntimeError): + # These origin/cursor errors are also fail-closed in maintained admission. + healthy = False + for name, path in paths.items(): + require(_read(path, absent=name != "enrollments") == snapshots[name], "impact_metadata_changed") + return healthy + + +def _impact_directory_valid(policy): + try: + admission._bound_directory(policy["sessions_dir"]) + return True + except ValueError as error: + if not str(error).startswith("admission_"): + raise + return False + except OSError: + return False + + +def _activation_impact(root, expected_policy_raw): + policy_raw, identity = _read(root / "admission.json") + require(policy_raw == expected_policy_raw, "impact_policy_changed") + policy = admission._json(policy_raw) + require(policy["mode"] == MODE_V1, "impact_requires_v1_policy") + members = _impact_members(root) + healthy = [] + # A strict v1 directory mismatch already prevents every v1 task from being + # admitted. Re-evaluate it on apply so an origin becoming valid is detected. + directory_valid = _impact_directory_valid(policy) + if directory_valid: + for sid in members: + if _impact_task(root, policy, policy_raw, sid): + healthy.append(sid) + require(len(healthy) <= MAX_SELECTED, "impact_healthy_cohort_limit_exceeded") + require(_impact_members(root) == members, "impact_membership_changed") + require(_impact_directory_valid(policy) == directory_valid, "impact_directory_changed") + require(_read(root / "admission.json") == (policy_raw, identity), "impact_policy_changed") + return {"schema_version": 1, "predicate": "host_v1_future_event_admission_v1", "policy_sha256": digest(policy_raw), + "policy_enabled": policy["enabled"], "legacy_sessions_directory_valid": directory_valid, + "healthy_session_ids": healthy, "max_tasks": MAX_IMPACT_TASKS} + + +def _validate_impact(plan): + original_raw = _unb64(plan["records"][-1]["preimage"]) + policy = admission._json(original_raw) + impact = plan["activation_impact"] + if policy["mode"] == MODE_V2: + require(impact is None, "unexpected_v2_impact_assertion") + return + require(isinstance(impact, dict) and set(impact) == {"schema_version", "predicate", "policy_sha256", "policy_enabled", + "legacy_sessions_directory_valid", "healthy_session_ids", "max_tasks"} + and type(impact["schema_version"]) is int and impact["schema_version"] == 1 + and impact["predicate"] == "host_v1_future_event_admission_v1" + and impact["policy_sha256"] == digest(original_raw) + and type(impact["policy_enabled"]) is bool and impact["policy_enabled"] == policy["enabled"] + and type(impact["legacy_sessions_directory_valid"]) is bool + and type(impact["max_tasks"]) is int and impact["max_tasks"] == MAX_IMPACT_TASKS, "invalid_impact_assertion") + ids = impact["healthy_session_ids"] + require(isinstance(ids, list) and ids == sorted(set(ids)) and len(ids) <= MAX_SELECTED + and set(ids) <= set(plan["session_ids"]), "impact_healthy_cohort_not_selected") + + +def _recheck_impact(root, plan): + if plan["activation_impact"] is not None: + require(_activation_impact(root, _unb64(plan["records"][-1]["preimage"])) == plan["activation_impact"], + "impact_healthy_cohort_changed") + + +def _impact_seal(plan, plan_sha): + return encode({"schema_version": 1, "kind": "pre_transition_activation_impact", "plan_sha256": plan_sha, + "activation_impact": plan["activation_impact"]}) + + def _build(root, route_path, session_ids, legacy_policy_path): policy_raw, policy_info = _read(root / "admission.json") current = admission._json(policy_raw) @@ -344,6 +486,8 @@ def _build(root, route_path, session_ids, legacy_policy_path): route_raw, route_info = _read(route_path) route = admission._json(route_raw) _route(route, root, stable) + impact = None if stable else _activation_impact(root, policy_raw) + require(impact is None or set(impact["healthy_session_ids"]) <= set(session_ids), "impact_healthy_cohort_not_selected") if legacy_policy_path is None: require(not stable, "retained_legacy_policy_required") legacy_raw, legacy_info = policy_raw, policy_info @@ -382,6 +526,7 @@ def _build(root, route_path, session_ids, legacy_policy_path): return {"schema_version": 1, "kind": PLAN_KIND, "root": str(root), "route_path": str(route_path), "session_ids": session_ids, "runtime_sha256": _runtime(), "records": records, "hold_paths": holds, "transitional_policy": _record(root / "admission.json", policy_raw, policy_info, transition_raw), + "activation_impact": impact, "legacy_policy": {"path": str(legacy_path), "bytes": _b64(legacy_raw), "sha256": digest(legacy_raw), "identity": legacy_info}, "historical_volume_continuity_proven": False, "owner_authorized_current_volume_adoption": True, "learner_started": False, "publication_order": ORDER} @@ -408,7 +553,7 @@ def _validate_plan(root, plan, expected_sha): and digest(encode(plan)) == expected_sha, "plan_sha_mismatch") require(set(plan) == {"schema_version", "kind", "root", "route_path", "session_ids", "runtime_sha256", "records", "hold_paths", "legacy_policy", "historical_volume_continuity_proven", - "owner_authorized_current_volume_adoption", "learner_started", "publication_order", "transitional_policy"} + "owner_authorized_current_volume_adoption", "learner_started", "publication_order", "transitional_policy", "activation_impact"} and type(plan["schema_version"]) is int and plan["schema_version"] == 1 and plan["kind"] == PLAN_KIND and plan["root"] == str(root) and plan["publication_order"] == ORDER @@ -446,6 +591,7 @@ def _validate_plan(root, plan, expected_sha): transition = encode({**original, "enabled": False}) if original["enabled"] else original_raw require(plan["transitional_policy"] == _record(root / "admission.json", original_raw, plan["records"][-1]["preimage_identity"], transition), "transitional_policy_changed") + _validate_impact(plan) def _validate_candidates(root, plan): @@ -581,7 +727,35 @@ def _run(root, plan, plan_sha, *, recovery): with Locks(root) as locks: _validate_plan(root, plan, plan_sha) states = _verify_prefix(plan, recover=recovery) - work = _journal(root, plan, plan_sha, create=not recovery) + if not recovery: + # This must precede even journal creation: changed eligibility causes + # zero application mutations, not merely zero task-record writes. + _recheck_impact(root, plan) + locks.verify() + work = _journal(root, plan, plan_sha, create=True) + else: + work = _journal(root, plan, plan_sha, create=False) + if plan["activation_impact"] is not None: + seal_path = work / "activation-impact.json" + seal_raw, _ = _read(seal_path, absent=True) + expected_seal = _impact_seal(plan, plan_sha) + if seal_raw is None: + # Journal preparation may have been interrupted before the seal. + # It is never safe to reconstruct it after transition/hold writes. + policy_raw, info = _read(root / "admission.json") + require(policy_raw == _unb64(plan["records"][-1]["preimage"]) + and info == plan["records"][-1]["preimage_identity"] + and all(state == "before" for state in states) + and not any(os.path.lexists(path) for path in plan["hold_paths"]), "impact_seal_missing_after_transition") + if recovery: + _recheck_impact(root, plan) + _new(seal_path, expected_seal) + else: + require(seal_raw == expected_seal, "impact_seal_changed") + # A crash before an enabled original policy was suspended leaves + # enrollment possible; recheck that cohort before continuing. + if recovery and _policy_phase(plan, recover=True) == "original": + _recheck_impact(root, plan) if _policy_phase(plan, recover=True) == "original": locks.verify() _event(work, "before_transitional_policy", candidate_sha256=plan["transitional_policy"]["candidate_sha256"]) @@ -844,7 +1018,10 @@ def deadline(*_): result = (apply if args.command == "apply" else recover)(args.state_dir, plan, args.plan_sha256) if args.output is not None: _new(args.output, encode(result)) - print(json.dumps({"status": result.get("status", "review_required"), "sha256": digest(encode(result)), "learner_started": False})) + summary = {"status": result.get("status", "review_required"), "sha256": digest(encode(result)), "learner_started": False} + if result.get("activation_impact") is not None: + summary["reviewed_healthy_session_ids"] = result["activation_impact"]["healthy_session_ids"] + print(json.dumps(summary)) return 0 except (OSError, ValueError, TypeError, KeyError) as error: reason = str(error) if isinstance(error, ValueError) and re.fullmatch(r"(?:admission|migration)_[a-z_]+", str(error)) else "migration_failed" From 815411cfb92ddf1cd96762d85604e7800385ab99 Mon Sep 17 00:00:00 2001 From: jsflax Date: Mon, 21 Sep 2026 15:17:27 -0400 Subject: [PATCH 4/8] Support bounded 64-task identity migrations without repeated full cohort validation --- Tests/codex_plugin/test_migration.py | 396 +++++++++++++++++- codex/plugins/engram/README.md | 14 +- .../engram/scripts/codex_learner/migration.py | 70 +++- 3 files changed, 459 insertions(+), 21 deletions(-) diff --git a/Tests/codex_plugin/test_migration.py b/Tests/codex_plugin/test_migration.py index 7aa30df..a61c669 100644 --- a/Tests/codex_plugin/test_migration.py +++ b/Tests/codex_plugin/test_migration.py @@ -118,7 +118,7 @@ def test_explicit_authorization_and_bounded_selection(self): with self.assertRaisesRegex(ValueError, "invalid_selection"): self.prepare([self.sid, self.sid]) with self.assertRaisesRegex(ValueError, "invalid_selection"): - self.prepare([F.uuid7(F.ORIGIN, suffix=n) for n in range(33)]) + self.prepare([F.uuid7(F.ORIGIN, suffix=n) for n in range(M.MAX_SELECTED + 1)]) def test_apply_preserves_every_nonidentity_field_and_other_task(self): other = self.fixture.task(2) @@ -804,8 +804,8 @@ def test_existing_hold_cannot_hide_retained_v1_healthy_task(self): self.prepare([self.sid, other]) self.assertEqual(before, self.snapshot()) - def test_healthy_cohort_over_32_refuses_without_expansion_or_mutation(self): - for n in range(2, 35): + def test_healthy_cohort_over_selected_bound_refuses_without_expansion_or_mutation(self): + for n in range(2, M.MAX_SELECTED + 3): self.fixture.task(n, cursor=False, pending=False) before = self.snapshot() with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_limit_exceeded"): @@ -890,5 +890,395 @@ def test_later_v2_selection_keeps_policy_hash_without_v1_rescan(self): self.assertEqual((self.root / "admission.json").read_bytes(), policy_raw) +class Bound64Tests(unittest.TestCase): + """Expanded selection boundary with real owned metadata and unchanged guards.""" + setUp = MigrationTests.setUp + prepare = MigrationTests.prepare + apply = MigrationTests.apply + snapshot = MigrationTests.snapshot + healthy = ImpactTests.healthy + unchanged_refusal = ImpactTests.unchanged_refusal + + def cohort(self, *, first_healthy=True): + if first_healthy: + self.healthy(self.sid) + selected = [self.sid] + for n in range(2, 65): + cursor = n % 4 != 0 + selected.append(self.fixture.task(n, cursor=cursor, pending=cursor, + paused=n % 3 != 0, gated=False)) + self.assertEqual(len(selected), 64) + return sorted(selected) + + def test_exact_64_healthy_tasks_prepare_without_mutation_or_changed_other_bounds(self): + self.assertEqual(M.MAX_SELECTED, 64) + self.assertEqual(M.MAX_IMPACT_TASKS, 500) + self.assertEqual(M.MAX_PLAN_BYTES, 32 * 1024 * 1024) + selected = self.cohort() + before = self.snapshot() + plan = self.prepare(selected) + self.assertEqual(plan["session_ids"], selected) + self.assertEqual(plan["activation_impact"]["healthy_session_ids"], selected) + self.assertEqual(len(plan["records"]), 64 * 4 + 2) + self.assertLess(len(M.encode(plan)), M.MAX_PLAN_BYTES) + M._validate_plan(self.root, plan, M.digest(M.encode(plan))) + self.assertEqual(before, self.snapshot()) + self.assertFalse((self.root / "migrations").exists()) + self.assertFalse((self.root / "migration-holds").exists()) + + def test_64_plan_preserves_all_cursor_gate_pause_and_dedup_fields(self): + selected = self.cohort(first_healthy=False) + plan = self.prepare(selected) + self.assertEqual(len(plan["activation_impact"]["healthy_session_ids"]), 63) + found_gate = found_paused = found_unpaused = found_event_bridge = False + for record in plan["records"]: + name = Path(record["path"]).parent.name + if name not in {"sessions", "pending", "admissions"} or record["preimage"] is None: + continue + before = json.loads(M._unb64(record["preimage"])) + after = json.loads(M._unb64(record["candidate"])) + if name == "admissions": + self.assertEqual({k: v for k, v in before.items() if k not in {"binding", "state_sha256"}}, + {k: v for k, v in after.items() if k not in {"binding", "state_sha256", "legacy_event_identity"}}) + self.assertEqual(after["last_event"], before["last_event"]) + self.assertEqual(after["processing"], before["processing"]) + self.assertEqual(after["legacy_event_identity"], + {k: before["binding"][k] for k in ("device", "inode")}) + found_event_bridge = True + continue + self.assertEqual({k: v for k, v in before.items() if k not in {"device", "inode", "admission"}}, + {k: v for k, v in after.items() if k not in {"identity", "admission"}}) + found_gate |= bool(after.get("reconciliation_required")) + if name == "pending": + found_paused |= after.get("paused_request_id") == after["request_id"] + found_unpaused |= after.get("paused_request_id") != after["request_id"] + self.assertTrue(found_gate and found_paused and found_unpaused and found_event_bridge) + + def test_65_explicit_selected_tasks_refuse_before_mutation(self): + selected = self.cohort() + selected.append(self.fixture.task(65, cursor=False, pending=False)) + before = self.snapshot() + with self.assertRaisesRegex(ValueError, "invalid_selection"): + self.prepare(selected) + self.assertEqual(before, self.snapshot()) + self.assertFalse((self.root / "migrations").exists()) + + def test_65_healthy_tasks_cannot_fit_by_omitting_one_from_64_selection(self): + selected = self.cohort() + self.fixture.task(65, cursor=False, pending=False) + before = self.snapshot() + with self.assertRaisesRegex(ValueError, "impact_healthy_cohort_limit_exceeded"): + self.prepare(selected) + self.assertEqual(before, self.snapshot()) + + def test_new_65th_healthy_enrollment_after_64_plan_refuses_without_journal(self): + selected = self.cohort() + plan = self.prepare(selected) + self.fixture.task(65, cursor=False, pending=False) + self.unchanged_refusal(plan, "impact_healthy_cohort_limit_exceeded") + self.assertEqual(plan["session_ids"], selected) + + def test_new_membership_within_health_bound_still_refuses_exact_64_selection(self): + selected = self.cohort(first_healthy=False) + plan = self.prepare(selected) + self.fixture.task(65, cursor=False, pending=False) + self.unchanged_refusal(plan) + + def test_existing_gate_clear_outside_64_selection_refuses_without_mutation(self): + selected = self.cohort(first_healthy=False) + outsider = self.fixture.task(65, gated=True) + plan = self.prepare(selected) + self.healthy(outsider) + self.unchanged_refusal(plan) + + def test_existing_processing_completion_outside_64_selection_refuses_without_mutation(self): + selected = self.cohort(first_healthy=False) + outsider = self.fixture.task(65, gated=False) + self.healthy(outsider, processing="existing-run") + plan = self.prepare(selected) + self.healthy(outsider, processing=None) + self.unchanged_refusal(plan) + + def test_last_selected_cursor_stale_cas_still_refuses_at_64(self): + selected = self.cohort() + plan = self.prepare(selected) + cursor_sid = next(sid for sid in reversed(selected) if M._paths(self.root, sid)["sessions"].exists()) + path = M._paths(self.root, cursor_sid)["sessions"] + state = json.loads(path.read_bytes()) + state["newer_state_must_survive"] = True + F.write_json(path, state) + self.unchanged_refusal(plan, "cas_mismatch") + + +class PublicationEfficiencyTests(unittest.TestCase): + """Scoped native revalidation still protects every actual publication.""" + setUp = MigrationTests.setUp + prepare = MigrationTests.prepare + apply = MigrationTests.apply + snapshot = MigrationTests.snapshot + healthy = ImpactTests.healthy + + def cohort(self): + self.healthy(self.sid) + return [self.sid] + [self.fixture.task(n, gated=False) for n in range(2, 65)] + + def extend_frontier(self, sid): + paths = M._paths(self.root, sid) + entry = json.loads(paths["enrollments"].read_bytes()) + entry.update(M.admission._frontier(entry)) + F.write_json(paths["enrollments"], entry) + binding = M._binding(entry, self.fixture.policy, (self.root / "admission.json").read_bytes()) + state = json.loads(paths["sessions"].read_bytes()) + state.update(admission=binding, offset=entry["frontier_offset"]) + F.write_json(paths["sessions"], state) + record = json.loads(paths["admissions"].read_bytes()) + record.update(binding=binding, state_sha256=M.object_digest(state)) + F.write_json(paths["admissions"], record) + pending = json.loads(paths["pending"].read_bytes()) + pending["admission"] = binding + F.write_json(paths["pending"], pending) + + def assert_suspended(self): + self.assertIs(json.loads((self.root / "admission.json").read_bytes())["enabled"], False) + + def test_late_task_frontier_drift_refuses_before_that_task_publication(self): + other = self.fixture.task(2, gated=False) + self.extend_frontier(other) + plan = self.prepare([self.sid, other]) + late = self.fixture.tasks[other]["path"] + original = M._event + def change(work, event, **fields): + original(work, event, **fields) + if event == "published" and fields["index"] == 3: + raw = late.read_bytes() + self.assertIn(b"fixture durable fact", raw) + late.write_bytes(raw.replace(b"fixture durable fact", b"changed durable fact")) + with mock.patch.object(M, "_event", side_effect=change): + with self.assertRaisesRegex(ValueError, "legacy_frontier_changed"): + self.apply(plan) + self.assert_suspended() + for record in plan["records"][4:8]: + self.assertEqual(Path(record["path"]).read_bytes(), M._unb64(record["preimage"])) + + def test_late_task_origin_drift_refuses_before_that_task_publication(self): + other = self.fixture.task(2, gated=False) + plan = self.prepare([self.sid, other]) + late = self.fixture.tasks[other]["path"] + original = M._event + def change(work, event, **fields): + original(work, event, **fields) + if event == "published" and fields["index"] == 3: + late.write_bytes(late.read_bytes().replace(b'"cli_version": "fixture"', b'"cli_version": "changed"')) + with mock.patch.object(M, "_event", side_effect=change): + with self.assertRaisesRegex(ValueError, "legacy_origin_changed"): + self.apply(plan) + self.assert_suspended() + self.assertEqual(Path(plan["records"][4]["path"]).read_bytes(), M._unb64(plan["records"][4]["preimage"])) + + def test_full_final_validation_rechecks_already_published_task_before_activation(self): + other = self.fixture.task(2, gated=False) + self.extend_frontier(self.sid) + plan = self.prepare([self.sid, other]) + path = self.fixture.tasks[self.sid]["path"] + original = M._event + def change(work, event, **fields): + original(work, event, **fields) + if event == "published" and fields["index"] == len(plan["records"]) - 2: + path.write_bytes(path.read_bytes().replace(b"fixture durable fact", b"changed durable fact")) + with mock.patch.object(M, "_event", side_effect=change): + with self.assertRaisesRegex(ValueError, "legacy_frontier_changed"): + self.apply(plan) + self.assert_suspended() + self.assertEqual(self.route.read_bytes(), M._unb64(plan["records"][-2]["candidate"])) + self.assertNotEqual((self.root / "admission.json").read_bytes(), M._unb64(plan["records"][-1]["candidate"])) + + def test_runtime_drift_after_first_write_refuses_next_write(self): + plan = self.prepare() + original_event, original_runtime = M._event, M._runtime + changed = False + def change(work, event, **fields): + nonlocal changed + original_event(work, event, **fields) + if event == "published" and fields["index"] == 0: + changed = True + def runtime(): + actual = original_runtime() + return {**actual, "changed.py": "0" * 64} if changed else actual + with mock.patch.object(M, "_event", side_effect=change), mock.patch.object(M, "_runtime", side_effect=runtime): + with self.assertRaisesRegex(ValueError, "runtime_changed"): + self.apply(plan) + self.assert_suspended() + self.assertEqual(Path(plan["records"][1]["path"]).read_bytes(), M._unb64(plan["records"][1]["preimage"])) + + def test_global_prefix_cas_still_checks_previously_published_task(self): + other = self.fixture.task(2, gated=False) + plan = self.prepare([self.sid, other]) + first = M._paths(self.root, self.sid)["sessions"] + original = M._event + def change(work, event, **fields): + original(work, event, **fields) + if event == "published" and fields["index"] == 3: + value = json.loads(first.read_bytes()) + F.write_json(first, {**value, "newer": "must remain untouched"}) + with mock.patch.object(M, "_event", side_effect=change): + with self.assertRaisesRegex(ValueError, "cas_mismatch"): + self.apply(plan) + self.assert_suspended() + self.assertEqual(json.loads(first.read_bytes())["newer"], "must remain untouched") + + def test_recovery_skips_published_prefix_native_scans_and_completes_64_under_cli_deadline(self): + import time + selected = self.cohort() + plan = self.prepare(selected) + plan_path = self.base / "reviewed-plan.json" + plan_path.write_bytes(M.encode(plan)); plan_path.chmod(0o600) + sha = M.digest(M.encode(plan)) + original = M._publish + def crash(record, work, index): + original(record, work, index) + if index == 239: + raise RuntimeError("injected after 60 complete tasks") + with mock.patch.object(M, "_publish", side_effect=crash): + with self.assertRaisesRegex(RuntimeError, "60 complete tasks"): + self.apply(plan) + before = {r["path"]: Path(r["path"]).read_bytes() for r in plan["records"][:240]} + candidate_calls = [] + original_validate = M._validate_candidates + def validate(*args, **kwargs): + candidate_calls.append(kwargs.get("candidate_task_ids")) + return original_validate(*args, **kwargs) + started = time.monotonic() + with mock.patch.object(M, "_validate_candidates", side_effect=validate), mock.patch("builtins.print") as output: + result = M.main(["recover", "--state-dir", str(self.root), "--plan", str(plan_path), "--plan-sha256", sha]) + self.assertEqual(result, 0) + self.assertLess(time.monotonic() - started, 30) + self.assertEqual(json.loads(output.call_args.args[0])["status"], "migration_complete_held") + self.assertEqual(candidate_calls.count(None), 3) # Entry, pre-activation, completed chain. + subsets = [value for value in candidate_calls if value is not None] + self.assertEqual(subsets, [tuple([sid]) for sid in selected[60:] for _ in range(4)] + [()]) + self.assertEqual(before, {path: Path(path).read_bytes() for path in before}) + self.assertEqual((self.root / "admission.json").read_bytes(), M._unb64(plan["records"][-1]["candidate"])) + + def test_completed_recovery_still_revalidates_all_sources_and_origins(self): + plan = self.prepare() + self.apply(plan) + source = M._runtime() + with mock.patch.object(M, "_runtime", return_value={**source, "changed.py": "0" * 64}): + with self.assertRaisesRegex(ValueError, "runtime_changed"): + M.recover(self.root, plan, M.digest(M.encode(plan))) + path = self.fixture.tasks[self.sid]["path"] + path.write_bytes(path.read_bytes().replace(b'"cli_version": "fixture"', b'"cli_version": "changed"')) + with self.assertRaisesRegex(ValueError, "legacy_origin_changed"): + M.recover(self.root, plan, M.digest(M.encode(plan))) + + +class PublicationPlanIntegrityTests(unittest.TestCase): + """Exact pinned bytes replace only repeated immutable parsing, never live checks.""" + setUp = MigrationTests.setUp + prepare = MigrationTests.prepare + apply = MigrationTests.apply + snapshot = MigrationTests.snapshot + assert_suspended = PublicationEfficiencyTests.assert_suspended + + def changed_plan_refuses(self, mutate): + plan = self.prepare() + sha = M.digest(M.encode(plan)) + original_event = M._event + before_next = Path(plan["records"][1]["path"]).read_bytes() + def change(work, event, **fields): + original_event(work, event, **fields) + if event == "published" and fields["index"] == 0: + mutate(plan) + with mock.patch.object(M, "_event", side_effect=change): + with self.assertRaisesRegex(ValueError, "plan_sha_mismatch"): + M.apply(self.root, plan, sha) + self.assert_suspended() + self.assertEqual(Path(plan["records"][1]["path"]).read_bytes(), before_next) + + def test_nested_record_mutation_after_first_write_refuses_pinned_sha(self): + def mutate(plan): + record = plan["records"][1] + state = json.loads(M._unb64(record["candidate"])) + state["offset"] += 1 + raw = M.encode(state) + record.update(candidate=M._b64(raw), candidate_sha256=M.digest(raw)) + self.changed_plan_refuses(mutate) + + def test_nested_impact_mutation_after_first_write_refuses_pinned_sha(self): + self.changed_plan_refuses(lambda plan: plan["activation_impact"].update(policy_enabled=False)) + + def test_nested_transition_mutation_after_first_write_refuses_pinned_sha(self): + self.changed_plan_refuses(lambda plan: plan["transitional_policy"]["preimage_identity"].update(mtime_ns=0)) + + def test_cli_rehashed_invalid_plan_cannot_skip_full_entry_validation(self): + plan = self.prepare() + plan["records"][1]["candidate_sha256"] = "0" * 64 + plan_path = self.base / "altered-plan.json" + plan_path.write_bytes(M.encode(plan)) + plan_path.chmod(0o600) + before = self.snapshot() + with mock.patch.object(M, "_validate_publication") as publication, mock.patch("builtins.print") as output: + result = M.main(["apply", "--state-dir", str(self.root), "--plan", str(plan_path), + "--plan-sha256", M.digest(M.encode(plan))]) + self.assertNotEqual(result, 0) + self.assertIn("record_digest_changed", output.call_args.args[0]) + publication.assert_not_called() + self.assertEqual(self.snapshot(), before) + + def later_plan(self): + second = self.fixture.task(2) + first = self.prepare() + self.apply(first) + journal = self.root / "migrations" / M.digest(M.encode(first)) + later = self.prepare([second], legacy_policy_path=journal / "legacy-policy.json") + return later, journal + + def assert_later_write_refuses(self, plan, change_source, reason): + original_event = M._event + def change(work, event, **fields): + original_event(work, event, **fields) + if event == "published" and fields["index"] == 0: + change_source() + with mock.patch.object(M, "_event", side_effect=change): + with self.assertRaisesRegex(ValueError, reason): + self.apply(plan) + self.assert_suspended() + self.assertEqual(Path(plan["records"][1]["path"]).read_bytes(), M._unb64(plan["records"][1]["preimage"])) + + def test_retained_legacy_identity_drift_after_first_write_refuses(self): + plan, journal = self.later_plan() + source = journal / "legacy-policy.json" + self.assert_later_write_refuses(plan, lambda: source.write_bytes(source.read_bytes()), "legacy_source_changed") + + def test_retained_journal_lineage_drift_after_first_write_refuses(self): + plan, journal = self.later_plan() + source = journal / "plan.json" + def change(): + value = json.loads(source.read_bytes()) + value["learner_started"] = True + source.write_bytes(M.encode(value)) + self.assert_later_write_refuses(plan, change, "retained_plan_digest_changed") + + def test_later_stable_policy_reencoding_refuses_before_any_write(self): + plan, _ = self.later_plan() + record = plan["records"][-1] + raw = (json.dumps(json.loads(M._unb64(record["candidate"])), indent=2) + "\n").encode() + record.update(candidate=M._b64(raw), candidate_sha256=M.digest(raw)) + before = self.snapshot() + with self.assertRaisesRegex(ValueError, "stable_policy_must_not_change"): + self.apply(plan) + self.assertEqual(self.snapshot(), before) + + def test_immutable_impact_validation_only_at_full_boundaries(self): + plan = self.prepare() + with mock.patch.object(M, "_validate_impact", wraps=M._validate_impact) as immutable, \ + mock.patch.object(M, "_validate_publication", wraps=M._validate_publication) as publication: + self.apply(plan) + self.assertEqual(immutable.call_count, 3) # Entry, pre-activation, completed chain. + self.assertEqual(publication.call_count, 5) # Four task records, then the route. + self.assertEqual([call.kwargs["candidate_task_ids"] for call in publication.call_args_list], + [(self.sid,)] * 4 + [()]) + + if __name__ == "__main__": unittest.main() diff --git a/codex/plugins/engram/README.md b/codex/plugins/engram/README.md index dbbb30e..9dab830 100644 --- a/codex/plugins/engram/README.md +++ b/codex/plugins/engram/README.md @@ -99,15 +99,23 @@ as such; migration does not prove historical volume continuity. The first host policy conversion must include the explicitly reviewed healthy v1 tasks, including tasks waiting for their next event. The tool rechecks that cohort under its locks before changing state. A newly eligible task causes a -refusal and a new review; it is never silently added. If the required cohort -exceeds the supported selection bound, keep v1 until a coordinated rollout is -prepared. Installing the new runtime preserves an existing v1 policy. +refusal and a new review; it is never silently added. A plan can explicitly select +at most 64 tasks; the impact scan remains bounded to 500 enrollment entries. +If the required cohort exceeds 64, keep v1 until a coordinated rollout is +prepared. Installing the new runtime preserves an existing v1 policy. A larger +selection still requires a fresh reviewed list of IDs and an exact plan; raising +the bound does not expand any existing selection or authorize activation. Applying a plan temporarily disables host admission while publishing the selected records and route, then restores the policy's original enabled setting in v2. The journal supports `recover` after an interrupted publication. Other tasks' records remain untouched; unmigrated v1 enrollments under v2 require their own migration. Later plans use the retained original v1 policy as their legacy anchor. +The 32 MiB plan limit and 30-second command deadline remain unchanged. An oversized +plan is refused before migration begins. A deadline after publication has started +can leave admission suspended or tasks held; use the exact journal's `recover` +path after review. Planning speed alone does not qualify application or recovery +time for a larger cohort. Each migrated task receives a separate migration hold. `release-plan` and `release` remove only explicitly selected holds after validating their records. diff --git a/codex/plugins/engram/scripts/codex_learner/migration.py b/codex/plugins/engram/scripts/codex_learner/migration.py index e2a3e91..c1dd819 100644 --- a/codex/plugins/engram/scripts/codex_learner/migration.py +++ b/codex/plugins/engram/scripts/codex_learner/migration.py @@ -23,7 +23,7 @@ from . import admission, file_identity, host_admission -MAX_SELECTED = 32 +MAX_SELECTED = 64 MAX_IMPACT_TASKS = 500 MAX_PLAN_BYTES = 32 * 1024 * 1024 MODE_V1 = "host_sessions_v1" @@ -548,9 +548,31 @@ def prepare(root, route_path, session_ids, *, legacy_policy_path=None, authorize return plan -def _validate_plan(root, plan, expected_sha): - require(isinstance(plan, dict) and len(encode(plan)) <= MAX_PLAN_BYTES - and digest(encode(plan)) == expected_sha, "plan_sha_mismatch") +def _validate_plan_digest(plan, expected_sha): + require(isinstance(plan, dict), "plan_sha_mismatch") + raw = encode(plan) + require(len(raw) <= MAX_PLAN_BYTES and digest(raw) == expected_sha, "plan_sha_mismatch") + + +def _validate_legacy_source(root, plan): + """Re-read external retained bytes and their exact journal lineage.""" + legacy = plan["legacy_policy"] + legacy_path = Path(legacy["path"]) + if legacy_path != root / "admission.json": + raw, identity = _read(legacy_path) + require(raw == _unb64(legacy["bytes"]) and identity == legacy["identity"], "legacy_source_changed") + if admission._json(_unb64(plan["records"][-1]["preimage"]))["mode"] == MODE_V2: + _retained_legacy(root, legacy_path, raw, _unb64(plan["records"][-1]["preimage"])) + + +def _validate_plan(root, plan, expected_sha, *, candidate_task_ids=None): + """Validate every immutable/source/shared assertion; default is every task. + + During one locked publication only, the caller may additionally narrow fresh + task-origin/frontier checks to the affected task. Full checks remain required + at command entry and before the final activation policy is published. + """ + _validate_plan_digest(plan, expected_sha) require(set(plan) == {"schema_version", "kind", "root", "route_path", "session_ids", "runtime_sha256", "records", "hold_paths", "legacy_policy", "historical_volume_continuity_proven", "owner_authorized_current_volume_adoption", "learner_started", "publication_order", "transitional_policy", "activation_impact"} @@ -579,13 +601,8 @@ def _validate_plan(root, plan, expected_sha): == (record["preimage_identity"] is None), "record_absence_changed") legacy = plan["legacy_policy"] require(digest(_unb64(legacy["bytes"])) == legacy["sha256"], "legacy_policy_digest_changed") - legacy_path = Path(legacy["path"]) - if legacy_path != root / "admission.json": - raw, identity = _read(legacy_path) - require(raw == _unb64(legacy["bytes"]) and identity == legacy["identity"], "legacy_source_changed") - if admission._json(_unb64(plan["records"][-1]["preimage"]))["mode"] == MODE_V2: - _retained_legacy(root, legacy_path, raw, _unb64(plan["records"][-1]["preimage"])) - _validate_candidates(root, plan) + _validate_legacy_source(root, plan) + _validate_candidates(root, plan, candidate_task_ids=candidate_task_ids) original_raw = _unb64(plan["records"][-1]["preimage"]) original = admission._json(original_raw) transition = encode({**original, "enabled": False}) if original["enabled"] else original_raw @@ -594,7 +611,21 @@ def _validate_plan(root, plan, expected_sha): _validate_impact(plan) -def _validate_candidates(root, plan): +def _validate_publication(root, plan, expected_sha, *, candidate_task_ids): + """Only for _run after full validation within its current lock acquisition. + + The pinned canonical digest proves that all immutable plan assertions remain + exactly those already validated at entry. Recheck every source and shared + live assertion, plus the affected task, before the caller's full prefix/hold + CAS. Nothing is cached across writes, commands, or lock acquisitions. + """ + _validate_plan_digest(plan, expected_sha) + require(plan["runtime_sha256"] == _runtime(), "runtime_changed") + _validate_legacy_source(root, plan) + _validate_candidates(root, plan, candidate_task_ids=candidate_task_ids) + + +def _validate_candidates(root, plan, *, candidate_task_ids=None): records = plan["records"] original = admission._json(_unb64(records[-1]["preimage"])) _policy(original, root, original.get("mode") == MODE_V2) @@ -616,7 +647,12 @@ def _validate_candidates(root, plan): _route(route_before, root, original["mode"] == MODE_V2) _route(route_after, root, True) require(route_after == {**route_before, "schema_version": 2, "mode": MODE_V2}, "route_transformation_invalid") + if candidate_task_ids is not None: + require(isinstance(candidate_task_ids, tuple) and len(candidate_task_ids) <= 1 + and set(candidate_task_ids) <= set(plan["session_ids"]), "invalid_candidate_task_selection") for index, sid in enumerate(plan["session_ids"]): + if candidate_task_ids is not None and sid not in candidate_task_ids: + continue batch = records[index * 4:index * 4 + 4] before = {Path(r["path"]).parent.name: _unb64(r["preimage"]) for r in batch} expected = _task_candidates(root, sid, before, legacy, legacy_raw, target, target_raw) @@ -773,13 +809,17 @@ def _run(root, plan, plan_sha, *, recovery): _new(path, expected) _event(work, "hold_published", session_id=sid) for index, record in enumerate(plan["records"][:-1]): + # Entry validation already proved the complete plan and exact prefix. + # Recovery must reach its first remaining record without rechecking + # every task's native identity once per already-published record. + if record["preimage"] == record["candidate"] or states[index] == "after": + continue locks.verify() - _validate_plan(root, plan, plan_sha) + affected = (plan["session_ids"][index // 4],) if index < len(plan["session_ids"]) * 4 else () + _validate_publication(root, plan, plan_sha, candidate_task_ids=affected) _verify_prefix(plan, recover=True) for sid, path in zip(plan["session_ids"], plan["hold_paths"]): require(_read(Path(path))[0] == _hold(sid, plan_sha), "hold_changed") - if record["preimage"] == record["candidate"] or _match(record, recover=True) == "after": - continue _event(work, "before_publish", index=index, path=record["path"], candidate_sha256=record["candidate_sha256"]) _publish(record, work, index) _event(work, "published", index=index, path=record["path"], candidate_sha256=record["candidate_sha256"]) From a125cd761852133258a1104a924ba95b48919a6f Mon Sep 17 00:00:00 2001 From: jsflax Date: Mon, 21 Sep 2026 21:04:17 -0400 Subject: [PATCH 5/8] fix: keep remember writes in checked transaction and classify startup timeouts --- Sources/EngramKit/MemoryTools+Core.swift | 73 ++++--- .../LatticeAdoptionRecallTests.swift | 196 +++++++++++++++++- Tests/codex_plugin/test_receipts.py | 77 +++++++ .../engram/scripts/codex_learner/runner.py | 26 ++- 4 files changed, 335 insertions(+), 37 deletions(-) diff --git a/Sources/EngramKit/MemoryTools+Core.swift b/Sources/EngramKit/MemoryTools+Core.swift index af97b21..14a6fd6 100644 --- a/Sources/EngramKit/MemoryTools+Core.swift +++ b/Sources/EngramKit/MemoryTools+Core.swift @@ -187,59 +187,58 @@ extension MemoryTools { autoConnectCandidates = Array(autoConnectCandidates.prefix(3)) } - // Episode: end stale episodes on 30-min gap - if activeEpisodeId != nil { - let gap = Date().timeIntervalSince(lastMemoryTime) - if gap > 1800 { endActiveEpisode() } - } - lastMemoryTime = Date() - - let isPrivate = a.isPrivate ?? false - - log("[remember] creating Memory object") - let memory = Memory(content: content, topic: topic, project: project, source: source, embedding: embeddingVec, expiresAt: expiresAt, importance: importance, isPrivate: isPrivate, authorUserId: currentUserId, modifiedAt: Date()) - log("[remember] calling localLattice.add()") - try localLattice.add(memory) - log("[remember] add() complete") - - lastRememberedId = memory.globalId - guard let memoryGlobalId = memory.globalId else { - throw MCPError.internalError("Failed to persist memory — globalId is nil after add()") - } - - // Auto-connect, link, and organize — all writes in one transaction - var parentNote = "" - var autoLinkedGids: [UUID] = [] let parentGidValue: UUID? = a.parentId?.value - if let parentGid = parentGidValue { guard findMemory(id: parentGid) != nil else { throw MCPError.invalidParams("parent_id \(parentGid.uuidString) not found") } } - try localLattice.transaction { + // Prepare episode bookkeeping without publishing it on a failed write. + // There is no suspension between this snapshot and the checked commit. + let rememberedAt = Date() + let episodeExpired = activeEpisodeId != nil && rememberedAt.timeIntervalSince(lastMemoryTime) > 1800 + let episodeId = episodeExpired ? nil : activeEpisodeId + let isPrivate = a.isPrivate ?? false + let memory = Memory(content: content, topic: topic, project: project, source: source, embedding: embeddingVec, expiresAt: expiresAt, importance: importance, isPrivate: isPrivate, authorUserId: currentUserId, modifiedAt: rememberedAt) + var parentNote = "" + var autoLinkedGids: [UUID] = [] + var committedLogs: [String] = [] + + // The Memory and its required graph/topic effects share one checked + // transaction. A failed BEGIN cannot execute the body in autocommit. + // Other errors still propagate without claiming that rollback settled. + let memoryGlobalId = try localLattice.withTransaction { () throws -> UUID in + if let parentGid = parentGidValue { + guard findMemory(id: parentGid) != nil else { + throw MCPError.invalidParams("parent_id \(parentGid.uuidString) not found") + } + } + try localLattice.add(memory) + guard let memoryGlobalId = memory.globalId else { + throw MCPError.internalError("Failed to persist memory — globalId is nil after add()") + } // Auto-create part_of edge when parent_id is provided if let parentGid = parentGidValue { let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: parentGid, relation: .partOf, authorUserId: currentUserId) try localLattice.add(edge) parentNote = ", parent: \(parentGid.uuidString)" - log("Auto-created part_of edge: \(memoryGlobalId.uuidString) -> \(parentGid.uuidString)") + committedLogs.append("Auto-created part_of edge: \(memoryGlobalId.uuidString) -> \(parentGid.uuidString)") } // Link to active episode via part_of edge - if let epGid = activeEpisodeId, + if let epGid = episodeId, localLattice.objects(Memory.self).where({ $0.globalId == epGid }).first != nil { let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: epGid, relation: .partOf, authorUserId: currentUserId) try localLattice.add(edge) - log("Linked memory \(memoryGlobalId.uuidString) to episode \(epGid.uuidString)") + committedLogs.append("Linked memory \(memoryGlobalId.uuidString) to episode \(epGid.uuidString)") } // Auto-connect: create relates_to edges to semantically similar memories for candidate in autoConnectCandidates { guard let candidateGlobalId = candidate.object.globalId else { continue } if let pgid = parentGidValue, candidateGlobalId == pgid { continue } - if let epGid = activeEpisodeId, candidateGlobalId == epGid { continue } + if let epGid = episodeId, candidateGlobalId == epGid { continue } let forwardEdge = localLattice.objects(Edge.self) .where { $0.sourceGlobalId == memoryGlobalId && $0.targetGlobalId == candidateGlobalId && $0.relation == .relatesTo } .first != nil @@ -251,7 +250,7 @@ extension MemoryTools { let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: candidateGlobalId, relation: .relatesTo, authorUserId: currentUserId) try localLattice.add(edge) autoLinkedGids.append(candidateGlobalId) - log("Auto-connected [\(memoryGlobalId.uuidString)] --[relates_to]--> [\(candidateGlobalId.uuidString)] (distance: \(String(format: "%.3f", candidate.distance)))") + committedLogs.append("Auto-connected [\(memoryGlobalId.uuidString)] --[relates_to]--> [\(candidateGlobalId.uuidString)] (distance: \(String(format: "%.3f", candidate.distance)))") } // Cross-project hub linking @@ -297,7 +296,7 @@ extension MemoryTools { let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: hubGlobalId, relation: .relatesTo, authorUserId: currentUserId) try localLattice.add(edge) autoLinkedGids.append(hubGlobalId) - log("Cross-project link [\(memoryGlobalId.uuidString)] --[relates_to]--> [\(hubGlobalId.uuidString)] (project '\(otherProject)' mentioned in content)") + committedLogs.append("Cross-project link [\(memoryGlobalId.uuidString)] --[relates_to]--> [\(hubGlobalId.uuidString)] (project '\(otherProject)' mentioned in content)") } } @@ -339,7 +338,7 @@ extension MemoryTools { if !alreadyLinked { let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: hubGlobalId, relation: .partOf, authorUserId: currentUserId) try localLattice.add(edge) - log("Auto-organized [\(memoryGlobalId.uuidString)] into hub [\(hubGlobalId.uuidString)]") + committedLogs.append("Auto-organized [\(memoryGlobalId.uuidString)] into hub [\(hubGlobalId.uuidString)]") } } } @@ -348,11 +347,19 @@ extension MemoryTools { let (consensusTopic, count) = topicCounts.max(by: { $0.value < $1.value }), count >= 2 { memory.topic = consensusTopic - log("Auto-inferred topic '\(consensusTopic)' for [\(memoryGlobalId.uuidString)]") + committedLogs.append("Auto-inferred topic '\(consensusTopic)' for [\(memoryGlobalId.uuidString)]") } } + return memoryGlobalId } + // Publish process-local state and success logs only after checked commit + // returns. A body/commit/notification failure leaves these untouched. + if episodeExpired { endActiveEpisode() } + lastMemoryTime = rememberedAt + lastRememberedId = memoryGlobalId + for message in committedLogs { log(message) } + let expiresNote = expiresAt == .distantFuture ? "" : ", expires: \(Self.dateFormatter.string(from: expiresAt))" let importanceNote = importance > 0 ? ", importance: \(importance)" : "" let privateNote = isPrivate ? ", private: true" : "" diff --git a/Tests/EngramTests/LatticeAdoptionRecallTests.swift b/Tests/EngramTests/LatticeAdoptionRecallTests.swift index f067ffc..d91cc90 100644 --- a/Tests/EngramTests/LatticeAdoptionRecallTests.swift +++ b/Tests/EngramTests/LatticeAdoptionRecallTests.swift @@ -1,8 +1,9 @@ -import EngramKit +@testable import EngramKit import EngramMemoryCore import EngramModels import Foundation import Lattice +import MCP import SQLite3 import Testing @@ -115,6 +116,7 @@ struct LatticeAdoptionRecallTests { private struct SavedMemory { let id: UUID let content: String + let topic: String let accesses: Int let accessedAt: Double } @@ -147,9 +149,26 @@ struct LatticeAdoptionRecallTests { } } + func edgeCount() throws -> Int { + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM Edge", -1, &statement, nil) == SQLITE_OK, + let statement else { + throw SQLiteFailure(message: String(cString: sqlite3_errmsg(db))) + } + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { + throw SQLiteFailure(message: String(cString: sqlite3_errmsg(db))) + } + let count = Int(sqlite3_column_int64(statement, 0)) + guard sqlite3_step(statement) == SQLITE_DONE else { + throw SQLiteFailure(message: String(cString: sqlite3_errmsg(db))) + } + return count + } + func memories() throws -> [UUID: SavedMemory] { var statement: OpaquePointer? - guard sqlite3_prepare_v2(db, "SELECT globalId, content, accessCount, lastAccessedAt FROM Memory", -1, &statement, nil) == SQLITE_OK, + guard sqlite3_prepare_v2(db, "SELECT globalId, content, accessCount, lastAccessedAt, topic FROM Memory", -1, &statement, nil) == SQLITE_OK, let statement else { throw SQLiteFailure(message: String(cString: sqlite3_errmsg(db))) } @@ -159,10 +178,12 @@ struct LatticeAdoptionRecallTests { while status == SQLITE_ROW { guard let idText = sqlite3_column_text(statement, 0), let id = UUID(uuidString: String(cString: idText)), - let content = sqlite3_column_text(statement, 1) else { + let content = sqlite3_column_text(statement, 1), + let topic = sqlite3_column_text(statement, 4) else { throw SQLiteFailure(message: "Invalid persisted Memory identity/content") } rows[id] = SavedMemory(id: id, content: String(cString: content), + topic: String(cString: topic), accesses: Int(sqlite3_column_int64(statement, 2)), accessedAt: sqlite3_column_double(statement, 3)) status = sqlite3_step(statement) @@ -317,4 +338,173 @@ struct LatticeAdoptionRecallTests { #expect(saved[recovered.id]?.content == "recovered durable write") #expect(!saved.values.contains { $0.content == "write must fail" }) } + + @Test + func remember_missingParentDoesNotInsertMemoryOrPublishBookkeeping() async throws { + let fixture = try fixture(seed: false) + let previousTime = Self.oldAccess + await fixture.tools.setLastMemoryTime(previousTime) + await #expect(throws: (any Error).self) { + _ = try await fixture.tools.handle(CallTool.Parameters(name: "remember", arguments: [ + "content": .string("a rejected parent must not leave an orphan memory"), + "project": .string("Persistence"), + "parent_id": .string(UUID().uuidString), + ])) + } + let saved = try SQL(fixture.path, readOnly: true) + #expect(try saved.memories().isEmpty) + #expect(try saved.edgeCount() == 0) + let lastTime = await fixture.tools.lastMemoryTime + let lastId = await fixture.tools.lastRememberedId + #expect(lastTime == previousTime) + #expect(lastId == nil) + } + + @Test(arguments: ["edge-abort", "edge-rollback", "busy-begin"]) + func remember_graphFailurePreservesMemoryAndBookkeepingUntilCommit(_ failure: String) async throws { + let fixture = try fixture(seed: false) + _ = try await fixture.tools.handle(CallTool.Parameters(name: "begin_episode", arguments: [ + "title": .string("atomic remember fixture"), "project": .string("Persistence"), + ])) + let activeEpisode = await fixture.tools.activeEpisodeId + let parentId = try #require(activeEpisode) + _ = try await fixture.tools.handle(CallTool.Parameters(name: "remember", arguments: [ + "content": .string("baseline remembered child"), "project": .string("Persistence"), + "topic": .string("episode"), + ])) + let previousId = await fixture.tools.lastRememberedId + #expect(previousId != nil) + let previousTime = Self.oldAccess + await fixture.tools.setLastMemoryTime(previousTime) + let originalMemories = try SQL(fixture.path, readOnly: true).memories() + let originalEdges = try SQL(fixture.path, readOnly: true).edgeCount() + let blocker = try SQL(fixture.path) + if failure == "busy-begin" { + try blocker.execute("BEGIN IMMEDIATE") + } else { + let action = failure == "edge-abort" ? "ABORT" : "ROLLBACK" + try blocker.execute("CREATE TRIGGER remember_edge_fault BEFORE INSERT ON Edge BEGIN SELECT RAISE(\(action), 'remember-edge-fault'); END") + } + defer { + try? blocker.execute(failure == "busy-begin" ? "ROLLBACK" : "DROP TRIGGER IF EXISTS remember_edge_fault") + } + let started = Date() + await #expect(throws: (any Error).self) { + _ = try await fixture.tools.handle(CallTool.Parameters(name: "remember", arguments: [ + "content": .string("child must roll back with its required parent edge"), + "project": .string("Persistence"), "topic": .string("episode"), + "parent_id": .string(parentId.uuidString), + ])) + } + #expect(Date().timeIntervalSince(started) < 5) + // New independent readers prove committed state, rather than inspecting + // the failed transaction's managed Memory instance or cached results. + let afterFailure = try SQL(fixture.path, readOnly: true) + #expect(Set(try afterFailure.memories().keys) == Set(originalMemories.keys)) + #expect(try afterFailure.edgeCount() == originalEdges) + let episodeAfterFailure = await fixture.tools.activeEpisodeId + let timeAfterFailure = await fixture.tools.lastMemoryTime + let idAfterFailure = await fixture.tools.lastRememberedId + #expect(episodeAfterFailure == parentId) + #expect(timeAfterFailure == previousTime) + #expect(idAfterFailure == previousId) + + try blocker.execute(failure == "busy-begin" ? "ROLLBACK" : "DROP TRIGGER remember_edge_fault") + _ = try await fixture.tools.handle(CallTool.Parameters(name: "remember", arguments: [ + "content": .string("subsequent child commits with its required parent edge"), + "project": .string("Persistence"), "topic": .string("episode"), + "parent_id": .string(parentId.uuidString), + ])) + let committedId = await fixture.tools.lastRememberedId + let id = try #require(committedId) + let saved = try SQL(fixture.path, readOnly: true) + let rows = try saved.memories() + #expect(Set(rows.keys) == Set(originalMemories.keys).union([id])) + #expect(rows[id]?.content == "subsequent child commits with its required parent edge") + #expect(try saved.edgeCount() == originalEdges + 1) + let episodeAfterCommit = await fixture.tools.activeEpisodeId + let timeAfterCommit = await fixture.tools.lastMemoryTime + #expect(episodeAfterCommit == nil) + #expect(timeAfterCommit > previousTime) + } + + @Test(arguments: ["ABORT", "ROLLBACK"]) + func remember_laterEdgeFailureRollsBackEarlierEdge(_ action: String) async throws { + let fixture = try fixture(seed: false) + let parent = Memory(content: "explicit parent", topic: "episode", project: "Persistence") + try fixture.writer.add(parent) + let parentId = try #require(parent.globalId) + _ = try await fixture.tools.handle(CallTool.Parameters(name: "begin_episode", arguments: [ + "title": .string("active episode"), "project": .string("Persistence"), + ])) + let episodeId = await fixture.tools.activeEpisodeId + let episode = try #require(episodeId) + let previousTime = Date() + await fixture.tools.setLastMemoryTime(previousTime) + let original = try SQL(fixture.path, readOnly: true).memories() + let fault = try SQL(fixture.path) + // The explicit parent edge succeeds first; the active episode edge + // then fails, testing rollback of both Memory and an earlier Edge. + try fault.execute("CREATE TRIGGER remember_later_edge_fault BEFORE INSERT ON Edge WHEN NEW.targetGlobalId = '\(episode.uuidString.lowercased())' BEGIN SELECT RAISE(\(action), 'remember-later-edge-fault'); END") + defer { try? fault.execute("DROP TRIGGER IF EXISTS remember_later_edge_fault") } + await #expect(throws: (any Error).self) { + _ = try await fixture.tools.handle(CallTool.Parameters(name: "remember", arguments: [ + "content": .string("child requires both edges"), "topic": .string("episode"), + "project": .string("Persistence"), "parent_id": .string(parentId.uuidString), + ])) + } + let saved = try SQL(fixture.path, readOnly: true) + #expect(Set(try saved.memories().keys) == Set(original.keys)) + #expect(try saved.edgeCount() == 0) + let time = await fixture.tools.lastMemoryTime + let id = await fixture.tools.lastRememberedId + let active = await fixture.tools.activeEpisodeId + #expect(time == previousTime) + #expect(id == nil) + #expect(active == episode) + } + + @Test(arguments: ["ABORT", "ROLLBACK"]) + func remember_inferredTopicFailureRollsBackMemoryAndEdges(_ action: String) async throws { + let fixture = try fixture(seed: false) + var vector = [Float](repeating: 0, count: 384) + vector[0] = 1 - 0.75 * 0.75 / 2 + vector[1] = (1 - vector[0] * vector[0]).squareRoot() + for content in ["first related neighbor", "second related neighbor"] { + try fixture.writer.add(Memory(content: content, topic: "contract", project: "Persistence", + embedding: Vector(vector))) + } + let original = try SQL(fixture.path, readOnly: true).memories() + let previousTime = Self.oldAccess + await fixture.tools.setLastMemoryTime(previousTime) + let fault = try SQL(fixture.path) + // Topic inference is the final write after Memory and both relates_to + // edges. Its nonthrowing setter must still poison checked commit. + try fault.execute("CREATE TRIGGER remember_topic_fault BEFORE UPDATE OF topic ON Memory WHEN OLD.topic = 'general' AND NEW.topic = 'contract' BEGIN SELECT RAISE(\(action), 'remember-topic-fault'); END") + defer { try? fault.execute("DROP TRIGGER IF EXISTS remember_topic_fault") } + let arguments: [String: Value] = [ + "content": .string("new connected concept"), "project": .string("Persistence"), + ] + await #expect(throws: (any Error).self) { + _ = try await fixture.tools.handle(CallTool.Parameters(name: "remember", arguments: arguments)) + } + let failed = try SQL(fixture.path, readOnly: true) + #expect(Set(try failed.memories().keys) == Set(original.keys)) + #expect(try failed.edgeCount() == 0) + let time = await fixture.tools.lastMemoryTime + let id = await fixture.tools.lastRememberedId + #expect(time == previousTime) + #expect(id == nil) + + try fault.execute("DROP TRIGGER remember_topic_fault") + _ = try await fixture.tools.handle(CallTool.Parameters(name: "remember", arguments: arguments)) + let committedId = await fixture.tools.lastRememberedId + let committed = try #require(committedId) + let saved = try SQL(fixture.path, readOnly: true) + let rows = try saved.memories() + #expect(Set(rows.keys) == Set(original.keys).union([committed])) + #expect(rows[committed]?.topic == "contract") + #expect(try saved.edgeCount() == 2) + } + } diff --git a/Tests/codex_plugin/test_receipts.py b/Tests/codex_plugin/test_receipts.py index e65162b..499538d 100644 --- a/Tests/codex_plugin/test_receipts.py +++ b/Tests/codex_plugin/test_receipts.py @@ -122,6 +122,83 @@ def test_update_receipt_requires_requested_exact_uuid(self): self.reply(1, f'Updated memory (id: {OTHER})') self.assertGreater(self.audit_result()['tool_errors'], 0) + def reconciliation(self, rows, *, complete=True): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) + raw = ''.join(json.dumps(row) + '\n' for row in rows) + (path / 'mcp-audit.jsonl').write_text(raw if complete else raw.rstrip('\n')) + return F.RUNNER.failure_reconciliation(path, True) + + def timeout_prefix(self): + return [{'event': 'relay_started'}, + {'event': 'initialize_compat', 'omitted_capability': 'codex/auth-change'}] + + def test_initialize_timeout_has_no_forwarded_write_but_is_not_success(self): + rows = [*self.timeout_prefix(), {'event': 'request_timeout'}] + self.assertIsNone(self.reconciliation(rows)) + self.audit.rows = rows + self.assertGreater(self.audit_result()['tool_errors'], 0) + + def test_timeout_retains_unknown_and_acknowledged_earlier_writes(self): + call = {'event': 'tool_call', 'id': 1, 'tool': 'remember'} + for result in (None, + {'event': 'tool_result', 'id': 1, 'tool': 'remember', + 'ok': False, 'forwarded': True, 'memory_ids': []}, + {'event': 'tool_result', 'id': 1, 'tool': 'remember', + 'ok': True, 'forwarded': True, 'memory_ids': [ID]}): + with self.subTest(result=result): + rows = [*self.timeout_prefix(), call] + if result is not None: + rows.append(result) + rows.append({'event': 'request_timeout'}) + gate = self.reconciliation(rows) + self.assertEqual(gate['reason'], 'successful_or_unverified_write') + self.assertEqual(gate['memory_ids'], [ID] if result and result['ok'] else []) + + def test_timeout_cleanup_can_record_failed_pending_call_but_not_erase_risk(self): + rows = [*self.timeout_prefix(), {'event': 'tool_call', 'id': 1, 'tool': 'remember'}, + {'event': 'request_timeout'}, + {'event': 'relay_finished', 'child_reaped': True, 'cleanup_overrun': False}, + {'event': 'tool_result', 'id': 1, 'tool': 'remember', + 'ok': False, 'forwarded': True, 'memory_ids': []}] + self.assertIsNotNone(self.reconciliation(rows)) + for forwarded in (False, None, 0, 1): + with self.subTest(forwarded=forwarded): + rows[-1]['forwarded'] = forwarded + self.assertEqual(self.reconciliation(rows)['reason'], 'write_status_unknown') + + def test_timeout_after_local_denial_does_not_create_write_uncertainty(self): + rows = [*self.timeout_prefix(), {'event': 'tool_call', 'id': 1, 'tool': 'remember'}, + {'event': 'tool_result', 'id': 1, 'tool': 'remember', + 'ok': False, 'forwarded': False, 'memory_ids': []}, + {'event': 'request_timeout'}] + self.assertIsNone(self.reconciliation(rows)) + + def test_malformed_or_nonterminal_timeout_remains_unknown(self): + timeout = {'event': 'request_timeout'} + cases = [[timeout], + [*self.timeout_prefix(), dict(timeout, no_writes=True)], + [*self.timeout_prefix(), timeout, timeout], + [*self.timeout_prefix(), timeout, {'event': 'relay_started'}], + [*self.timeout_prefix(), timeout, + {'event': 'tool_call', 'id': 1, 'tool': 'remember'}], + [*self.timeout_prefix(), timeout, {'event': 'initialize_compat'}], + [*self.timeout_prefix(), {'event': 'unknown_timeout'}]] + for rows in cases: + with self.subTest(rows=rows): + self.assertEqual(self.reconciliation(rows)['reason'], 'write_status_unknown') + self.assertEqual(self.reconciliation([*self.timeout_prefix(), timeout], complete=False)['reason'], + 'write_status_unknown') + + def test_duplicate_audit_fields_cannot_hide_a_write_as_a_timeout(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) + (path / 'mcp-audit.jsonl').write_text( + '{"event":"relay_started"}\n' + '{"event":"tool_call","event":"request_timeout"}\n') + self.assertEqual(F.RUNNER.failure_reconciliation(path, True)['reason'], + 'write_status_unknown') + def test_failed_and_denied_calls_consume_total_budget(self): self.call(1, 'recall'); self.reply(1, 'failed', error=True) self.call(2, 'shell') diff --git a/codex/plugins/engram/scripts/codex_learner/runner.py b/codex/plugins/engram/scripts/codex_learner/runner.py index 6a42a52..eb47a59 100644 --- a/codex/plugins/engram/scripts/codex_learner/runner.py +++ b/codex/plugins/engram/scripts/codex_learner/runner.py @@ -749,20 +749,44 @@ def failure_reconciliation(run_dir: Path, provider_started: bool | None) -> dict if provider_started is False: return None # The real runner proves no subprocess was created. unknown = {"reason": "write_status_unknown", "memory_ids": []} + def unique_fields(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate audit field") + value[key] = item + return value try: raw = memory_config.read_bytes(run_dir / "mcp-audit.jsonl", 1024 * 1024) if not raw or not raw.endswith(b"\n"): return unknown calls, results = {}, {} started = False + request_timed_out = False for line in raw.splitlines(): - entry = json.loads(line) + entry = json.loads(line, object_pairs_hook=unique_fields) if not isinstance(entry, dict): return unknown event = entry.get("event") if event == "relay_started": + if request_timed_out: + return unknown started = True continue + if event == "request_timeout": + # The trusted relay fsyncs each tool_call before forwarding it. + # This exact event terminates its message pump; cleanup may add + # failed pending results, but cannot forward another request. + # A timeout before any write therefore needs backoff, not a + # reconciliation gate. Earlier writes remain risky below. + if not started or request_timed_out or set(entry) != {"event"}: + return unknown + request_timed_out = True + continue + if request_timed_out and (event in {"tool_call", "initialize_compat"} + or (event == "tool_result" and + (entry.get("ok") is not False or entry.get("forwarded") is not True))): + return unknown if event in {"relay_finished", "relay_interrupted", "relay_failed", "relay_cleanup_failed", "initialize_compat"}: continue if event not in {"tool_call", "tool_result"}: From 0099a0236dcd8a54bfa0606b356e7aed5acc11c5 Mon Sep 17 00:00:00 2001 From: jsflax Date: Wed, 23 Sep 2026 19:24:10 -0400 Subject: [PATCH 6/8] Fix background callback crashes and recover host memory integration for 0.14.9 --- .github/workflows/release.yml | 23 +- CHANGELOG.md | 13 + .../xcshareddata/swiftpm/Package.resolved | 4 +- Package.resolved | 4 +- Sources/EngramKit/ClaudeMCPRegistration.swift | 156 ++++++++++ Sources/EngramKit/MemoryTools+Core.swift | 276 ++++++++++-------- Sources/EngramSceneKit/LockedSnapshot.swift | 26 ++ .../EngramVisualizer/App/CLIInstaller.swift | 38 +-- Sources/EngramVisualizer/Galaxy/Galaxy.swift | 12 +- .../LockedSnapshotTests.swift | 66 +++++ .../ClaudeMCPRegistrationTests.swift | 178 +++++++++++ .../LatticeAdoptionRecallTests.swift | 37 ++- Tests/codex_plugin/test_host.py | 34 +++ Tests/codex_plugin/test_receipts.py | 96 ++++++ .../plugins/engram/.codex-plugin/plugin.json | 2 +- .../scripts/codex_learner/memory_proxy.py | 54 +++- .../engram/scripts/codex_learner/runner.py | 7 +- release-train/README.md | 2 +- scripts/test_codex_release_workflow.py | 61 +++- 19 files changed, 896 insertions(+), 193 deletions(-) create mode 100644 Sources/EngramKit/ClaudeMCPRegistration.swift create mode 100644 Sources/EngramSceneKit/LockedSnapshot.swift create mode 100644 Tests/EngramSceneKitTests/LockedSnapshotTests.swift create mode 100644 Tests/EngramTests/ClaudeMCPRegistrationTests.swift diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c4beac..e111bf2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,8 +54,9 @@ jobs: APPLE_PASSWORD: ${{ secrets.APPLE_APP_PASSWORD }} SPARKLE: ${{ secrets.SPARKLE_PRIVATE_KEY }} SWIFTLM: ${{ secrets.SWIFTLM_CLONE_TOKEN }} + SLACK: ${{ secrets.SLACK_WEBHOOK_URL }} run: | - for key in CERT CERT_PASSWORD TEAM APPLE_ID APPLE_PASSWORD SPARKLE SWIFTLM; do + for key in CERT CERT_PASSWORD TEAM APPLE_ID APPLE_PASSWORD SPARKLE SWIFTLM SLACK; do if [ -z "${!key}" ]; then echo "::error::Required release credential unavailable: $key"; exit 1; fi done portable: @@ -135,7 +136,7 @@ jobs: python3 -B scripts/run_native_tests.py --diagnostics-dir build/native-test-diagnostics --timeout-seconds 1800 --silence-seconds 300 -- - swift test --force-resolved-versions --skip-build --filter "EngramTests|EngramMemoryCoreTests|EngramRealityKitTests|PositionVersionTests" + swift test --force-resolved-versions --skip-build --filter "EngramTests|EngramMemoryCoreTests|EngramRealityKitTests|PositionVersionTests|LockedSnapshotTests" --skip "PerfTests" --skip "keyBERTKeywordExtraction" --skip "recall_semanticRelevanceOrdering" @@ -454,6 +455,7 @@ jobs: # Direct dispatch runs the existing native pipeline; it does not depend on # a GITHUB_TOKEN-created tag triggering a second workflow. - name: Create GitHub Release + id: publish run: | if ! git show-ref --verify --quiet "refs/tags/$RELEASE_TAG"; then git tag "$RELEASE_TAG" "$EXPECTED_SHA"; fi git push origin "refs/tags/$RELEASE_TAG" @@ -499,7 +501,9 @@ jobs: git diff --cached --quiet || (git commit -m "Update engram to ${VERSION}" && git push) - name: Notify Slack - if: github.event_name == 'push' && success() && env.SLACK_WEBHOOK_URL != '' + # Both tag pushes and release-train dispatches announce a published release. + # A later appcast/tap failure must not suppress the announcement. + if: ${{ !cancelled() && steps.publish.outcome == 'success' }} env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} run: | @@ -520,7 +524,7 @@ jobs: # Escape for JSON CHANGES_ESCAPED=$(echo "$CHANGES" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read())[1:-1])') - curl -s -X POST "$SLACK_WEBHOOK_URL" \ + SLACK_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 30 -X POST "$SLACK_WEBHOOK_URL" \ -H 'Content-Type: application/json' \ -d "{ \"blocks\": [ @@ -538,18 +542,21 @@ jobs: } } ] - }" + }") + test "$SLACK_RESPONSE" = "ok" || { echo "::error::Slack did not acknowledge the release announcement"; exit 1; } + echo "Slack acknowledged the release announcement." # Failed releases were silent (the success notifier is if: success()) — # announce failures too, with a link to the run. - name: Notify Slack (failure) - if: github.event_name == 'push' && failure() && env.SLACK_WEBHOOK_URL != '' + if: failure() && steps.publish.outcome != 'success' && env.SLACK_WEBHOOK_URL != '' env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} run: | - curl -s -X POST "$SLACK_WEBHOOK_URL" \ + SLACK_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 30 -X POST "$SLACK_WEBHOOK_URL" \ -H 'Content-Type: application/json' \ - -d "{\"text\": \"❌ Engram ${RELEASE_TAG} release FAILED — ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" + -d "{\"text\": \"❌ Engram ${RELEASE_TAG} release FAILED — ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}") + test "$SLACK_RESPONSE" = "ok" || { echo "::error::Slack did not acknowledge the failure announcement"; exit 1; } - name: Cleanup keychain if: always() diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e5b088..a9995c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to Engram are documented in this file. ## [Unreleased] +## [0.14.9] - 2026-09-23 + +### Fixed +- Prevent background and active-use app crashes caused by repeated reads of Galaxy callbacks growing the invocation stack. +- Repair missing Claude Code memory MCP registration on app launch, including GUI launches without Claude on PATH; verify registration and retry failed setup without requiring an app upgrade. +- Keep existing customized MCP servers and other user configuration intact during registration repair. +- Keep memory and graph updates inside one checked transaction and publish bookkeeping only after commit. +- Let Codex learning recover on a later turn when a busy database prevents the write transaction from starting; preserve reconciliation for uncertain writes. +- Recognize verified near-duplicate responses without blocking future learning, and classify startup failures separately from uncertain writes. +- Preserve Codex task identity across APFS device renumbering and provide bounded migration with checks for other active tasks. +- Include published Lattice Core 2.0.7 fixes for concurrent row-ID generation, text bytes, attachment aliases, numeric defaults, and snapshot initialization. +- Send and verify the normal Slack release announcement for release-tool dispatches as well as tag pushes. + ## [0.14.8] - 2026-09-19 ### Fixed diff --git a/Engram.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Engram.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 6502c9d..cbe3419 100644 --- a/Engram.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Engram.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -87,8 +87,8 @@ "kind": "remoteSourceControl", "location": "https://github.com/jsflax/LatticeCore.git", "state": { - "revision": "559c496d7997677d06cc6d1f7a7a87abc440c210", - "version": "2.0.5" + "revision": "eb9a99c5d3945108ab9d9d0d39622616d82f3586", + "version": "2.0.7" } }, { diff --git a/Package.resolved b/Package.resolved index 660b553..78676ca 100644 --- a/Package.resolved +++ b/Package.resolved @@ -87,8 +87,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/jsflax/LatticeCore.git", "state" : { - "revision" : "559c496d7997677d06cc6d1f7a7a87abc440c210", - "version" : "2.0.5" + "revision" : "eb9a99c5d3945108ab9d9d0d39622616d82f3586", + "version" : "2.0.7" } }, { diff --git a/Sources/EngramKit/ClaudeMCPRegistration.swift b/Sources/EngramKit/ClaudeMCPRegistration.swift new file mode 100644 index 0000000..5fde692 --- /dev/null +++ b/Sources/EngramKit/ClaudeMCPRegistration.swift @@ -0,0 +1,156 @@ +import Foundation +import Darwin + +/// Repairs a missing user-scoped registration without replacing a user's server. +/// Call from a background queue: the CLI has a bounded wait, but config I/O is synchronous. +public enum ClaudeMCPRegistration { + public enum Outcome: Equatable, Sendable, CustomStringConvertible { + case alreadyRegistered, registered, preservedExistingServer, invalidConfiguration + case missingMemoryExecutable, claudeNotFound, launchFailed, verificationFailed + case commandFailed(Int32) + case timedOut(cleanupConfirmed: Bool) + + public var description: String { + switch self { + case .alreadyRegistered: "already registered" + case .registered: "registered and verified in user configuration" + case .preservedExistingServer: "existing memory server preserved; automatic replacement skipped" + case .invalidConfiguration: "user configuration could not be read safely; retrying on next app launch" + case .missingMemoryExecutable: "memory executable missing; retrying on next app launch" + case .claudeNotFound: "Claude executable not found; retrying on next app launch" + case .launchFailed: "Claude could not launch; retrying on next app launch" + case .verificationFailed: "CLI exited successfully but registration was not verified; retrying on next app launch" + case .commandFailed(let status): "Claude exited \(status); retrying on next app launch" + case .timedOut(let confirmed): + "Claude registration timed out (process exit confirmed: \(confirmed)); retrying on next app launch" + } + } + } + + public static func register( + memoryExecutable: URL, + home: URL, + environment: [String: String], + timeout: TimeInterval = 10 + ) -> Outcome { + register(memoryExecutable: memoryExecutable, home: home, environment: environment, + timeout: timeout, candidates: executableCandidates(home: home, environment: environment)) + } + + // Injectable candidates keep tests entirely on private fake executables. + static func register( + memoryExecutable: URL, + home: URL, + environment: [String: String], + timeout: TimeInterval, + candidates: [URL] + ) -> Outcome { + let config = configurationURL(home: home, environment: environment) + switch registrationState(at: config, memoryExecutable: memoryExecutable) { + case .matching: return .alreadyRegistered + case .other: return .preservedExistingServer + case .invalid: return .invalidConfiguration + case .missing: break + } + guard isExecutable(memoryExecutable) else { return .missingMemoryExecutable } + guard let claude = candidates.first(where: isExecutable) else { return .claudeNotFound } + + var env = environment + env.removeValue(forKey: "CLAUDECODE") + env["HOME"] = home.path + env["PATH"] = executableCandidates(home: home, environment: environment) + .map { $0.deletingLastPathComponent().path }.joined(separator: ":") + + let process = Process() + process.executableURL = claude + process.arguments = ["mcp", "add", "--scope", "user", "--transport", "stdio", + "memory", "--", memoryExecutable.path] + process.environment = env + process.currentDirectoryURL = home + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + let finished = DispatchSemaphore(value: 0) + process.terminationHandler = { _ in finished.signal() } + do { try process.run() } catch { return .launchFailed } + // Darwin Foundation launches Process in a separate group. Verify that + // ownership before ever using a negative PID: never signal our app's group. + let childPID = process.processIdentifier + let ownedGroup = getpgid(childPID) == childPID ? childPID : nil + + if finished.wait(timeout: .now() + max(0, timeout)) == .timedOut { + guard let ownedGroup else { + // Fail conservatively if a future Foundation implementation does + // not isolate the child. Direct-child exit cannot prove tree cleanup. + if process.isRunning { process.terminate() } + if finished.wait(timeout: .now() + 1) == .timedOut { + if process.isRunning { kill(childPID, SIGKILL) } + _ = finished.wait(timeout: .now() + 1) + } + return .timedOut(cleanupConfirmed: false) + } + kill(-ownedGroup, SIGTERM) + if waitForExit(process, ownedGroup: ownedGroup, timeout: 1) { + return .timedOut(cleanupConfirmed: true) + } + kill(-ownedGroup, SIGKILL) + return .timedOut(cleanupConfirmed: waitForExit(process, ownedGroup: ownedGroup, timeout: 1)) + } + guard process.terminationStatus == 0 else { return .commandFailed(process.terminationStatus) } + guard registrationState(at: config, memoryExecutable: memoryExecutable) == .matching else { + return .verificationFailed + } + return .registered + } + + private static func waitForExit(_ process: Process, ownedGroup: pid_t, timeout: TimeInterval) -> Bool { + let deadline = DispatchTime.now() + timeout + repeat { + // Waiting for the direct child alone misses children of CLI wrappers. + let groupGone = kill(-ownedGroup, 0) == -1 && errno == ESRCH + if !process.isRunning && groupGone { return true } + if DispatchTime.now() >= deadline { return false } + usleep(10_000) + } while true + } + + static func executableCandidates(home: URL, environment: [String: String]) -> [URL] { + let paths = (environment["PATH"] ?? "").split(separator: ":").map(String.init) + + [home.appendingPathComponent(".local/bin").path, "/opt/homebrew/bin", "/usr/local/bin", "/usr/bin", "/bin"] + var seen = Set() + return paths.filter { $0.hasPrefix("/") && seen.insert($0).inserted } + .map { URL(fileURLWithPath: $0).appendingPathComponent("claude") } + } + + static func configurationURL(home: URL, environment: [String: String]) -> URL { + guard let custom = environment["CLAUDE_CONFIG_DIR"], !custom.isEmpty else { + return home.appendingPathComponent(".claude.json") + } + let path = custom.hasPrefix("~/") ? home.appendingPathComponent(String(custom.dropFirst(2))).path : custom + return URL(fileURLWithPath: path, relativeTo: home).standardizedFileURL.appendingPathComponent(".claude.json") + } + + private enum RegistrationState { case missing, matching, other, invalid } + + private static func registrationState(at url: URL, memoryExecutable: URL) -> RegistrationState { + guard FileManager.default.fileExists(atPath: url.path) else { return .missing } + guard let data = try? Data(contentsOf: url), + let root = try? JSONSerialization.jsonObject(with: data) as? [String: Any] + else { return .invalid } + guard let serversValue = root["mcpServers"] else { return .missing } + guard let servers = serversValue as? [String: Any] else { return .invalid } + guard let value = servers["memory"] else { return .missing } + guard let server = value as? [String: Any], + server["command"] as? String == memoryExecutable.path, + server["type"] == nil || server["type"] as? String == "stdio", + server["args"] == nil || (server["args"] as? [String]) == [] + else { return .other } + return .matching + } + + private static func isExecutable(_ url: URL) -> Bool { + var directory: ObjCBool = false + return FileManager.default.fileExists(atPath: url.path, isDirectory: &directory) + && !directory.boolValue && FileManager.default.isExecutableFile(atPath: url.path) + } +} diff --git a/Sources/EngramKit/MemoryTools+Core.swift b/Sources/EngramKit/MemoryTools+Core.swift index 14a6fd6..37c68f5 100644 --- a/Sources/EngramKit/MemoryTools+Core.swift +++ b/Sources/EngramKit/MemoryTools+Core.swift @@ -208,149 +208,177 @@ extension MemoryTools { // The Memory and its required graph/topic effects share one checked // transaction. A failed BEGIN cannot execute the body in autocommit. // Other errors still propagate without claiming that rollback settled. - let memoryGlobalId = try localLattice.withTransaction { () throws -> UUID in - if let parentGid = parentGidValue { - guard findMemory(id: parentGid) != nil else { - throw MCPError.invalidParams("parent_id \(parentGid.uuidString) not found") - } - } - try localLattice.add(memory) - guard let memoryGlobalId = memory.globalId else { - throw MCPError.internalError("Failed to persist memory — globalId is nil after add()") - } - // Auto-create part_of edge when parent_id is provided - if let parentGid = parentGidValue { - let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: parentGid, relation: .partOf, authorUserId: currentUserId) - try localLattice.add(edge) - parentNote = ", parent: \(parentGid.uuidString)" - committedLogs.append("Auto-created part_of edge: \(memoryGlobalId.uuidString) -> \(parentGid.uuidString)") - } - - // Link to active episode via part_of edge - if let epGid = episodeId, - localLattice.objects(Memory.self).where({ $0.globalId == epGid }).first != nil { - let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: epGid, relation: .partOf, authorUserId: currentUserId) - try localLattice.add(edge) - committedLogs.append("Linked memory \(memoryGlobalId.uuidString) to episode \(epGid.uuidString)") - } - - // Auto-connect: create relates_to edges to semantically similar memories - for candidate in autoConnectCandidates { - guard let candidateGlobalId = candidate.object.globalId else { continue } - if let pgid = parentGidValue, candidateGlobalId == pgid { continue } - if let epGid = episodeId, candidateGlobalId == epGid { continue } - let forwardEdge = localLattice.objects(Edge.self) - .where { $0.sourceGlobalId == memoryGlobalId && $0.targetGlobalId == candidateGlobalId && $0.relation == .relatesTo } - .first != nil - let reverseEdge = localLattice.objects(Edge.self) - .where { $0.sourceGlobalId == candidateGlobalId && $0.targetGlobalId == memoryGlobalId && $0.relation == .relatesTo } - .first != nil - guard !(forwardEdge || reverseEdge) else { continue } - - let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: candidateGlobalId, relation: .relatesTo, authorUserId: currentUserId) - try localLattice.add(edge) - autoLinkedGids.append(candidateGlobalId) - committedLogs.append("Auto-connected [\(memoryGlobalId.uuidString)] --[relates_to]--> [\(candidateGlobalId.uuidString)] (distance: \(String(format: "%.3f", candidate.distance)))") - } - - // Cross-project hub linking - if topic != "episode" { - let allProjects = Set( - localLattice.objects(Memory.self) - .snapshot() - .map(\.project) - ).subtracting([project, "global"]) - - for otherProject in allProjects { - guard otherProject.count >= 3 else { continue } - - let pattern = "\\b\(NSRegularExpression.escapedPattern(for: otherProject))\\b" - guard let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive), - regex.firstMatch(in: content, range: NSRange(content.startIndex..., in: content)) != nil - else { continue } - - let projectMemories = localLattice.objects(Memory.self) - .where { $0.project == otherProject && $0.topic != "episode" } - .snapshot() - - var bestHub: (mem: Memory, count: Int)? = nil - for mem in projectMemories { - let incomingCount = localLattice.objects(Edge.self) - .where { $0.targetGlobalId == mem.globalId && $0.relation == .partOf } - .count - if incomingCount > 0 && (bestHub == nil || incomingCount > bestHub!.count) { - bestHub = (mem: mem, count: incomingCount) - } + var transactionBodyEntered = false + let memoryGlobalId: UUID + do { + memoryGlobalId = try localLattice.withTransaction { () throws -> UUID in + transactionBodyEntered = true + if let parentGid = parentGidValue { + guard findMemory(id: parentGid) != nil else { + throw MCPError.invalidParams("parent_id \(parentGid.uuidString) not found") } + } + try localLattice.add(memory) + guard let memoryGlobalId = memory.globalId else { + throw MCPError.internalError("Failed to persist memory — globalId is nil after add()") + } + // Auto-create part_of edge when parent_id is provided + if let parentGid = parentGidValue { + let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: parentGid, relation: .partOf, authorUserId: currentUserId) + try localLattice.add(edge) + parentNote = ", parent: \(parentGid.uuidString)" + committedLogs.append("Auto-created part_of edge: \(memoryGlobalId.uuidString) -> \(parentGid.uuidString)") + } - guard let hub = bestHub, let hubGlobalId = hub.mem.globalId else { continue } + // Link to active episode via part_of edge + if let epGid = episodeId, + localLattice.objects(Memory.self).where({ $0.globalId == epGid }).first != nil { + let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: epGid, relation: .partOf, authorUserId: currentUserId) + try localLattice.add(edge) + committedLogs.append("Linked memory \(memoryGlobalId.uuidString) to episode \(epGid.uuidString)") + } - let forwardLinked = localLattice.objects(Edge.self) - .where { $0.sourceGlobalId == memoryGlobalId && $0.targetGlobalId == hubGlobalId && $0.relation == .relatesTo } + // Auto-connect: create relates_to edges to semantically similar memories + for candidate in autoConnectCandidates { + guard let candidateGlobalId = candidate.object.globalId else { continue } + if let pgid = parentGidValue, candidateGlobalId == pgid { continue } + if let epGid = episodeId, candidateGlobalId == epGid { continue } + let forwardEdge = localLattice.objects(Edge.self) + .where { $0.sourceGlobalId == memoryGlobalId && $0.targetGlobalId == candidateGlobalId && $0.relation == .relatesTo } .first != nil - let reverseLinked = localLattice.objects(Edge.self) - .where { $0.sourceGlobalId == hubGlobalId && $0.targetGlobalId == memoryGlobalId && $0.relation == .relatesTo } + let reverseEdge = localLattice.objects(Edge.self) + .where { $0.sourceGlobalId == candidateGlobalId && $0.targetGlobalId == memoryGlobalId && $0.relation == .relatesTo } .first != nil - guard !(forwardLinked || reverseLinked) else { continue } + guard !(forwardEdge || reverseEdge) else { continue } - let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: hubGlobalId, relation: .relatesTo, authorUserId: currentUserId) + let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: candidateGlobalId, relation: .relatesTo, authorUserId: currentUserId) try localLattice.add(edge) - autoLinkedGids.append(hubGlobalId) - committedLogs.append("Cross-project link [\(memoryGlobalId.uuidString)] --[relates_to]--> [\(hubGlobalId.uuidString)] (project '\(otherProject)' mentioned in content)") + autoLinkedGids.append(candidateGlobalId) + committedLogs.append("Auto-connected [\(memoryGlobalId.uuidString)] --[relates_to]--> [\(candidateGlobalId.uuidString)] (distance: \(String(format: "%.3f", candidate.distance)))") } - } - - // Incremental topic/hub inference from auto-connect neighbors - if !autoConnectCandidates.isEmpty { - var hubCounts: [UUID: Int] = [:] - var topicCounts: [String: Int] = [:] - for candidate in autoConnectCandidates { - guard let candidateGlobalId = candidate.object.globalId else { continue } - for edge in localLattice.objects(Edge.self) - .where({ $0.sourceGlobalId == candidateGlobalId && $0.relation == .partOf }) { - if localLattice.objects(Memory.self) - .where({ $0.globalId == edge.targetGlobalId && $0.topic != "episode" }).first != nil { - hubCounts[edge.targetGlobalId, default: 0] += 1 + // Cross-project hub linking + if topic != "episode" { + let allProjects = Set( + localLattice.objects(Memory.self) + .snapshot() + .map(\.project) + ).subtracting([project, "global"]) + + for otherProject in allProjects { + guard otherProject.count >= 3 else { continue } + + let pattern = "\\b\(NSRegularExpression.escapedPattern(for: otherProject))\\b" + guard let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive), + regex.firstMatch(in: content, range: NSRange(content.startIndex..., in: content)) != nil + else { continue } + + let projectMemories = localLattice.objects(Memory.self) + .where { $0.project == otherProject && $0.topic != "episode" } + .snapshot() + + var bestHub: (mem: Memory, count: Int)? = nil + for mem in projectMemories { + let incomingCount = localLattice.objects(Edge.self) + .where { $0.targetGlobalId == mem.globalId && $0.relation == .partOf } + .count + if incomingCount > 0 && (bestHub == nil || incomingCount > bestHub!.count) { + bestHub = (mem: mem, count: incomingCount) + } } - } - if candidate.object.topic != "episode" { - let hasIncoming = localLattice.objects(Edge.self) - .where { $0.targetGlobalId == candidateGlobalId && $0.relation == .partOf } + + guard let hub = bestHub, let hubGlobalId = hub.mem.globalId else { continue } + + let forwardLinked = localLattice.objects(Edge.self) + .where { $0.sourceGlobalId == memoryGlobalId && $0.targetGlobalId == hubGlobalId && $0.relation == .relatesTo } .first != nil - if hasIncoming { - hubCounts[candidateGlobalId, default: 0] += 1 - } - } - let t = candidate.object.topic - if t != "general" && t != "episode" { - topicCounts[t, default: 0] += 1 + let reverseLinked = localLattice.objects(Edge.self) + .where { $0.sourceGlobalId == hubGlobalId && $0.targetGlobalId == memoryGlobalId && $0.relation == .relatesTo } + .first != nil + guard !(forwardLinked || reverseLinked) else { continue } + + let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: hubGlobalId, relation: .relatesTo, authorUserId: currentUserId) + try localLattice.add(edge) + autoLinkedGids.append(hubGlobalId) + committedLogs.append("Cross-project link [\(memoryGlobalId.uuidString)] --[relates_to]--> [\(hubGlobalId.uuidString)] (project '\(otherProject)' mentioned in content)") } } - if let (hubGlobalId, count) = hubCounts.max(by: { $0.value < $1.value }), - count >= 2 { - let skipHub = parentGidValue.map { $0 == hubGlobalId } ?? false - if !skipHub { - let alreadyLinked = localLattice.objects(Edge.self) - .where { $0.sourceGlobalId == memoryGlobalId && $0.targetGlobalId == hubGlobalId && $0.relation == .partOf } - .first != nil - if !alreadyLinked { - let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: hubGlobalId, relation: .partOf, authorUserId: currentUserId) - try localLattice.add(edge) - committedLogs.append("Auto-organized [\(memoryGlobalId.uuidString)] into hub [\(hubGlobalId.uuidString)]") + // Incremental topic/hub inference from auto-connect neighbors + if !autoConnectCandidates.isEmpty { + var hubCounts: [UUID: Int] = [:] + var topicCounts: [String: Int] = [:] + + for candidate in autoConnectCandidates { + guard let candidateGlobalId = candidate.object.globalId else { continue } + for edge in localLattice.objects(Edge.self) + .where({ $0.sourceGlobalId == candidateGlobalId && $0.relation == .partOf }) { + if localLattice.objects(Memory.self) + .where({ $0.globalId == edge.targetGlobalId && $0.topic != "episode" }).first != nil { + hubCounts[edge.targetGlobalId, default: 0] += 1 + } + } + if candidate.object.topic != "episode" { + let hasIncoming = localLattice.objects(Edge.self) + .where { $0.targetGlobalId == candidateGlobalId && $0.relation == .partOf } + .first != nil + if hasIncoming { + hubCounts[candidateGlobalId, default: 0] += 1 + } + } + let t = candidate.object.topic + if t != "general" && t != "episode" { + topicCounts[t, default: 0] += 1 + } + } + + if let (hubGlobalId, count) = hubCounts.max(by: { $0.value < $1.value }), + count >= 2 { + let skipHub = parentGidValue.map { $0 == hubGlobalId } ?? false + if !skipHub { + let alreadyLinked = localLattice.objects(Edge.self) + .where { $0.sourceGlobalId == memoryGlobalId && $0.targetGlobalId == hubGlobalId && $0.relation == .partOf } + .first != nil + if !alreadyLinked { + let edge = Edge(sourceGlobalId: memoryGlobalId, targetGlobalId: hubGlobalId, relation: .partOf, authorUserId: currentUserId) + try localLattice.add(edge) + committedLogs.append("Auto-organized [\(memoryGlobalId.uuidString)] into hub [\(hubGlobalId.uuidString)]") + } } } - } - if topic == "general", - let (consensusTopic, count) = topicCounts.max(by: { $0.value < $1.value }), - count >= 2 { - memory.topic = consensusTopic - committedLogs.append("Auto-inferred topic '\(consensusTopic)' for [\(memoryGlobalId.uuidString)]") + if topic == "general", + let (consensusTopic, count) = topicCounts.max(by: { $0.value < $1.value }), + count >= 2 { + memory.topic = consensusTopic + committedLogs.append("Auto-inferred topic '\(consensusTopic)' for [\(memoryGlobalId.uuidString)]") + } } + return memoryGlobalId } - return memoryGlobalId + } catch { + // Only a failed BEGIN proves this attempt did not run any write. + // Identical text from a body, COMMIT, rollback, or notification + // failure must retain its unknown persistence outcome. + guard !transactionBodyEntered, + case LatticeError.transactionError(let detail) = error, + ["database is locked", "database is busy", "database table is locked", + "database schema is locked"].contains(where: { + detail == "Failed to begin transaction: " + $0 + }) else { throw error } + return CallTool.Result( + content: [.text("Memory was not stored: the database was busy before the write transaction started. Retry on a later turn.")], + structuredContent: .object([ + "engram_write_receipt": .object([ + "schema_version": .int(1), + "tool": .string("remember"), + "write_outcome": .string("not_stored_transaction_not_started"), + "reason": .string("database_busy"), + "memory_ids": .array([]), + ]), + ]), + isError: true + ) } // Publish process-local state and success logs only after checked commit diff --git a/Sources/EngramSceneKit/LockedSnapshot.swift b/Sources/EngramSceneKit/LockedSnapshot.swift new file mode 100644 index 0000000..0883646 --- /dev/null +++ b/Sources/EngramSceneKit/LockedSnapshot.swift @@ -0,0 +1,26 @@ +import os + +/// A replaceable value that can be copied out under a lock and used after unlocking. +public final class LockedSnapshot: Sendable { + // Keep the value inside a nominal state type. When an optional function is + // the lock's direct generic State, Swift can reabstract it through the inout + // argument to withLock and write another wrapper back on every read. Those + // wrappers accumulate until invoking the callback overflows the stack. + private struct State: Sendable { + var value: Value + } + + private let lock: OSAllocatedUnfairLock + + public init(_ value: Value) { + lock = OSAllocatedUnfairLock(initialState: State(value: value)) + } + + public func read() -> Value { + lock.withLock { $0.value } + } + + public func set(_ value: Value) { + lock.withLock { $0.value = value } + } +} diff --git a/Sources/EngramVisualizer/App/CLIInstaller.swift b/Sources/EngramVisualizer/App/CLIInstaller.swift index 71b5d4c..72b413a 100644 --- a/Sources/EngramVisualizer/App/CLIInstaller.swift +++ b/Sources/EngramVisualizer/App/CLIInstaller.swift @@ -23,13 +23,14 @@ enum CLIInstaller { let fm = FileManager.default guard fm.fileExists(atPath: cliDir.path) else { return } - // Independent of Claude and of the CLI version stamp: retry after a - // user installs Codex/Python, even when Engram itself is up to date. + // Retry host registration even when the binary version is current. + // A GUI launch may precede installation of Claude, Codex or Python. // Interpreter discovery and config I/O must not block app startup. // Schedule after any binary replacement below, so a fresh install's // memory command exists before the Codex registration is validated. defer { DispatchQueue.global(qos: .utility).async { + registerMCPServer() syncCodexSupport(from: cliDir) } } @@ -41,8 +42,6 @@ enum CLIInstaller { return } - let isFirstInstall = installedVersion == nil - do { try fm.createDirectory(atPath: installDir, withIntermediateDirectories: true) @@ -131,9 +130,6 @@ enum CLIInstaller { // Write version marker try bundledVersion.write(toFile: versionFile, atomically: true, encoding: .utf8) - // Register MCP server (re-register on upgrade to update path if needed) - registerMCPServer(isFirstInstall: isFirstInstall) - // Install CLAUDE.md instructions and hooks config installClaudeMD() installSettings() @@ -274,27 +270,15 @@ enum CLIInstaller { // MARK: - MCP Server Registration - private static func registerMCPServer(isFirstInstall: Bool) { - let env = cleanEnv() - - // Remove first to ensure clean state (matches install.sh behavior) - if !isFirstInstall { - let remove = Process() - remove.executableURL = URL(fileURLWithPath: "/usr/bin/env") - remove.arguments = ["claude", "mcp", "remove", "memory"] - remove.environment = env - try? remove.run() - remove.waitUntilExit() + private static func registerMCPServer() { + let outcome = ClaudeMCPRegistration.register( + memoryExecutable: URL(fileURLWithPath: installDir + "/memory"), + home: URL(fileURLWithPath: NSHomeDirectory()), + environment: cleanEnv() + ) + if outcome != .alreadyRegistered { + NSLog("Engram Claude MCP registration: %@", outcome.description) } - - let add = Process() - add.executableURL = URL(fileURLWithPath: "/usr/bin/env") - add.arguments = [ - "claude", "mcp", "add", "--scope", "user", "--transport", "stdio", - "memory", "--", "\(installDir)/memory" - ] - add.environment = env - try? add.run() } // MARK: - CLAUDE.md diff --git a/Sources/EngramVisualizer/Galaxy/Galaxy.swift b/Sources/EngramVisualizer/Galaxy/Galaxy.swift index 17063cf..bc6e038 100644 --- a/Sources/EngramVisualizer/Galaxy/Galaxy.swift +++ b/Sources/EngramVisualizer/Galaxy/Galaxy.swift @@ -123,12 +123,12 @@ actor Galaxy: Identifiable { // Lattice's background callback thread, outside this actor — without // observer-path filtering, every post-load memory appeared in BOTH the // personal and synced galaxies until restart. - private let nodeFilterLock = OSAllocatedUnfairLock<(@Sendable (Memory) -> Bool)?>(initialState: nil) + private let nodeFilterLock = LockedSnapshot<(@Sendable (Memory) -> Bool)?>(nil) nonisolated var nodeFilter: (@Sendable (Memory) -> Bool)? { - nodeFilterLock.withLock { $0 } + nodeFilterLock.read() } nonisolated func setNodeFilter(_ filter: (@Sendable (Memory) -> Bool)?) { - nodeFilterLock.withLock { $0 = filter } + nodeFilterLock.set(filter) } /// effectiveProject resolution (decision 13) for GROUP galaxies: members @@ -139,12 +139,12 @@ actor Galaxy: Identifiable { /// one per member. Nil (personal/synced) is identity. Same lock idiom as /// nodeFilter — the observer path reads it off-actor. private let projectResolverLock = - OSAllocatedUnfairLock<(@Sendable (_ author: UUID?, _ local: String) -> String)?>(initialState: nil) + LockedSnapshot<(@Sendable (_ author: UUID?, _ local: String) -> String)?>(nil) nonisolated var projectResolver: (@Sendable (UUID?, String) -> String)? { - projectResolverLock.withLock { $0 } + projectResolverLock.read() } nonisolated func setProjectResolver(_ resolver: (@Sendable (UUID?, String) -> String)?) { - projectResolverLock.withLock { $0 = resolver } + projectResolverLock.set(resolver) } /// Resolved display project for a memory in this galaxy. diff --git a/Tests/EngramSceneKitTests/LockedSnapshotTests.swift b/Tests/EngramSceneKitTests/LockedSnapshotTests.swift new file mode 100644 index 0000000..8d13360 --- /dev/null +++ b/Tests/EngramSceneKitTests/LockedSnapshotTests.swift @@ -0,0 +1,66 @@ +import Darwin +import Foundation +import Testing +import os +import EngramSceneKit + +@Suite("Locked callback snapshots") +struct LockedSnapshotTests { + private final class Input: @unchecked Sendable {} + private typealias Filter = @Sendable (Input) -> Bool + private typealias Resolver = @Sendable (UUID?, String) -> String + + @inline(never) private static func stackDepth() -> Int { + let frames = UnsafeMutablePointer.allocate(capacity: 4096) + defer { frames.deallocate() } + return Int(backtrace(frames, 4096)) + } + + @Test("Reading a stored node filter does not deepen its invocation stack") + func filterReadsKeepConstantStackDepth() { + let depths = OSAllocatedUnfairLock<[Int]>(initialState: []) + let callbacks = LockedSnapshot(nil) + let input = Input() + callbacks.set { _ in + let depth = Self.stackDepth() + depths.withLock { $0.append(depth) } + return true + } + for reads in [1, 32, 512] { + for _ in 0.. { _, _ in String(Self.stackDepth()) } + var observed: [Int] = [] + for reads in [1, 32, 512] { + for _ in 0..(nil) + let input = Input() + #expect(callbacks.read() == nil) + callbacks.set { _ in true } + let previous = callbacks.read() + callbacks.set { _ in false } + #expect(previous?(input) == true) + #expect(callbacks.read()?(input) == false) + callbacks.set(nil) + #expect(callbacks.read() == nil) + #expect(previous?(input) == true) + } +} diff --git a/Tests/EngramTests/ClaudeMCPRegistrationTests.swift b/Tests/EngramTests/ClaudeMCPRegistrationTests.swift new file mode 100644 index 0000000..457f154 --- /dev/null +++ b/Tests/EngramTests/ClaudeMCPRegistrationTests.swift @@ -0,0 +1,178 @@ +import Foundation +import Darwin +import Testing +@testable import EngramKit + +private struct MCPRegistrationFixture { + let home: URL + let memory: URL + let claude: URL + let config: URL + + init() throws { + home = FileManager.default.temporaryDirectory.appendingPathComponent("engram-mcp-\(UUID().uuidString)") + memory = home.appendingPathComponent(".claude/bin/memory") + claude = home.appendingPathComponent(".local/bin/claude") + config = home.appendingPathComponent(".claude.json") + try FileManager.default.createDirectory(at: memory.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: claude.deletingLastPathComponent(), withIntermediateDirectories: true) + try writeExecutable(memory, "#!/bin/sh\nexit 0\n") + } + + func cleanup() { try? FileManager.default.removeItem(at: home) } + + func writeExecutable(_ url: URL, _ contents: String) throws { + try contents.write(to: url, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: url.path) + } + + func writeConfig(_ object: [String: Any], to url: URL? = nil) throws { + try JSONSerialization.data(withJSONObject: object, options: .sortedKeys).write(to: url ?? config) + } + + var desiredConfig: [String: Any] { + ["untouched": "sentinel", "mcpServers": [ + "other": ["type": "http", "url": "https://example.invalid/mcp"], + "memory": ["type": "stdio", "command": memory.path, "args": []] + ]] + } + + func successfulClaude() throws { + try writeConfig(desiredConfig, to: home.appendingPathComponent("expected.json")) + try writeExecutable(claude, """ + #!/bin/sh + printf '%s\n' "$@" > "$HOME/arguments" + printf '%s' "${CLAUDECODE-unset}" > "$HOME/nesting" + /bin/cp "$HOME/expected.json" "${CLAUDE_CONFIG_DIR:-$HOME}/.claude.json" + """) + } + + func register(environment: [String: String] = [:], timeout: TimeInterval = 2, + candidates: [URL]? = nil) -> ClaudeMCPRegistration.Outcome { + ClaudeMCPRegistration.register(memoryExecutable: memory, home: home, environment: environment, + timeout: timeout, candidates: candidates ?? [claude]) + } +} + +@Test("GUI discovery includes native Claude and both Homebrew locations, without relative PATH entries") +func claudeMCPRegistrationDiscovery() { + let home = URL(fileURLWithPath: "/private/fixture-home") + let paths = ClaudeMCPRegistration.executableCandidates(home: home, environment: ["PATH": "/usr/bin:.:relative:/usr/bin:"]).map(\.path) + #expect(paths == ["/usr/bin/claude", "/private/fixture-home/.local/bin/claude", + "/opt/homebrew/bin/claude", "/usr/local/bin/claude", "/bin/claude"]) +} + +@Test("Missing registration is added in user scope and read back without removing any server") +func claudeMCPRegistrationAddsAndVerifies() throws { + let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } + try fixture.writeConfig(["untouched": "sentinel", "mcpServers": ["other": ["type": "http", "url": "https://example.invalid/mcp"]]]) + try fixture.successfulClaude() + #expect(fixture.register(environment: ["PATH": "/usr/bin:/bin", "CLAUDECODE": "1"]) == .registered) + let arguments = try String(contentsOf: fixture.home.appendingPathComponent("arguments"), encoding: .utf8) + #expect(arguments.components(separatedBy: .newlines) == ["mcp", "add", "--scope", "user", "--transport", "stdio", "memory", "--", fixture.memory.path, ""]) + #expect(try String(contentsOf: fixture.home.appendingPathComponent("nesting"), encoding: .utf8) == "unset") + #expect(fixture.register(candidates: []) == .alreadyRegistered) + let config = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: fixture.config)) as? [String: Any]) + #expect(config["untouched"] as? String == "sentinel") + #expect((config["mcpServers"] as? [String: Any])?["other"] != nil) +} + +@Test("An existing customized or external memory server is preserved byte for byte") +func claudeMCPRegistrationPreservesOtherServer() throws { + let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } + try fixture.successfulClaude() + for server: [String: Any] in [ + ["type": "http", "url": "https://example.invalid/memory"], + ["command": "/a/different/memory", "args": []], + ["command": fixture.memory.path, "args": ["--custom"]] + ] { + try fixture.writeConfig(["mcpServers": ["memory": server]]) + let before = try Data(contentsOf: fixture.config) + #expect(fixture.register() == .preservedExistingServer) + #expect(try Data(contentsOf: fixture.config) == before) + #expect(!FileManager.default.fileExists(atPath: fixture.home.appendingPathComponent("arguments").path)) + } +} + +@Test("Unreadable JSON and malformed MCP objects are never replaced") +func claudeMCPRegistrationPreservesMalformedConfiguration() throws { + let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } + try fixture.successfulClaude() + for contents in ["{broken", "[]", "{\"mcpServers\":null}"] { + try contents.write(to: fixture.config, atomically: true, encoding: .utf8) + #expect(fixture.register() == .invalidConfiguration) + #expect(try String(contentsOf: fixture.config, encoding: .utf8) == contents) + } + #expect(!FileManager.default.fileExists(atPath: fixture.home.appendingPathComponent("arguments").path)) +} + +@Test("Missing Claude and failed registrations can retry without changing the app version") +func claudeMCPRegistrationRetriesFailures() throws { + let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } + #expect(fixture.register(candidates: []) == .claudeNotFound) + try fixture.writeExecutable(fixture.claude, "#!/bin/sh\nexit 7\n") + #expect(fixture.register() == .commandFailed(7)) + try fixture.writeExecutable(fixture.claude, "#!/bin/sh\nexit 0\n") + #expect(fixture.register() == .verificationFailed) + try fixture.successfulClaude() + #expect(fixture.register() == .registered) +} + +@Test("A missing memory binary prevents registration") +func claudeMCPRegistrationRequiresMemory() throws { + let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } + try FileManager.default.removeItem(at: fixture.memory) + #expect(fixture.register(candidates: []) == .missingMemoryExecutable) +} + +@Test("Registration honors Claude's custom user configuration directory") +func claudeMCPRegistrationCustomConfigDirectory() throws { + let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } + let profile = fixture.home.appendingPathComponent("profile") + try FileManager.default.createDirectory(at: profile, withIntermediateDirectories: true) + try fixture.writeConfig(["mcpServers": ["memory": ["type": "http", "url": "https://example.invalid/keep"]]]) + let before = try Data(contentsOf: fixture.config) + try fixture.successfulClaude() + #expect(fixture.register(environment: ["CLAUDE_CONFIG_DIR": profile.path]) == .registered) + #expect(try Data(contentsOf: fixture.config) == before) + #expect(FileManager.default.fileExists(atPath: profile.appendingPathComponent(".claude.json").path)) +} + +@Test("A hung Claude command is stopped within a bounded wait") +func claudeMCPRegistrationTimeout() throws { + let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } + // exec keeps the sleeper in the exact child process, without a detached descendant. + try fixture.writeExecutable(fixture.claude, "#!/bin/sh\nexec /bin/sleep 30\n") + let start = Date() + #expect(fixture.register(timeout: 0.05) == .timedOut(cleanupConfirmed: true)) + #expect(Date().timeIntervalSince(start) < 3) +} + +@Test("A timed-out Claude wrapper and its TERM-resistant descendant are both stopped") +func claudeMCPRegistrationTimeoutCleansWrapperDescendant() throws { + let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } + let childFile = fixture.home.appendingPathComponent("descendant-pid") + let wrapperFile = fixture.home.appendingPathComponent("wrapper-pid") + try fixture.writeExecutable(fixture.claude, """ + #!/bin/sh + trap '' TERM + /bin/sleep 30 & + child=$! + printf '%s' "$child" > "$HOME/descendant-pid" + printf '%s' "$$" > "$HOME/wrapper-pid" + wait "$child" + """) + let start = Date() + let result = fixture.register(timeout: 0.25) + let child = try #require(Int32(String(contentsOf: childFile, encoding: .utf8))) + let wrapper = try #require(Int32(String(contentsOf: wrapperFile, encoding: .utf8))) + // Cleanup remains bounded even if the assertion detects a regression. + defer { + if kill(child, 0) == 0 { kill(child, SIGKILL) } + if kill(wrapper, 0) == 0 { kill(wrapper, SIGKILL) } + } + #expect(result == .timedOut(cleanupConfirmed: true)) + #expect(kill(child, 0) == -1 && errno == ESRCH) + #expect(kill(wrapper, 0) == -1 && errno == ESRCH) + #expect(Date().timeIntervalSince(start) < 3) +} diff --git a/Tests/EngramTests/LatticeAdoptionRecallTests.swift b/Tests/EngramTests/LatticeAdoptionRecallTests.swift index d91cc90..2d4e3a7 100644 --- a/Tests/EngramTests/LatticeAdoptionRecallTests.swift +++ b/Tests/EngramTests/LatticeAdoptionRecallTests.swift @@ -360,7 +360,7 @@ struct LatticeAdoptionRecallTests { #expect(lastId == nil) } - @Test(arguments: ["edge-abort", "edge-rollback", "busy-begin"]) + @Test(arguments: ["edge-abort", "edge-rollback", "edge-begin-spoof", "busy-begin"]) func remember_graphFailurePreservesMemoryAndBookkeepingUntilCommit(_ failure: String) async throws { let fixture = try fixture(seed: false) _ = try await fixture.tools.handle(CallTool.Parameters(name: "begin_episode", arguments: [ @@ -382,19 +382,40 @@ struct LatticeAdoptionRecallTests { if failure == "busy-begin" { try blocker.execute("BEGIN IMMEDIATE") } else { - let action = failure == "edge-abort" ? "ABORT" : "ROLLBACK" - try blocker.execute("CREATE TRIGGER remember_edge_fault BEFORE INSERT ON Edge BEGIN SELECT RAISE(\(action), 'remember-edge-fault'); END") + let action = failure == "edge-rollback" ? "ROLLBACK" : "ABORT" + let detail = failure == "edge-begin-spoof" ? "Failed to begin transaction: database is locked" : "remember-edge-fault" + try blocker.execute("CREATE TRIGGER remember_edge_fault BEFORE INSERT ON Edge BEGIN SELECT RAISE(\(action), '\(detail)'); END") } defer { try? blocker.execute(failure == "busy-begin" ? "ROLLBACK" : "DROP TRIGGER IF EXISTS remember_edge_fault") } let started = Date() - await #expect(throws: (any Error).self) { - _ = try await fixture.tools.handle(CallTool.Parameters(name: "remember", arguments: [ - "content": .string("child must roll back with its required parent edge"), - "project": .string("Persistence"), "topic": .string("episode"), - "parent_id": .string(parentId.uuidString), + let request = CallTool.Parameters(name: "remember", arguments: [ + "content": .string("child must roll back with its required parent edge"), + "project": .string("Persistence"), "topic": .string("episode"), + "parent_id": .string(parentId.uuidString), + ]) + if failure == "busy-begin" { + let result = try await fixture.tools.handle(request) + #expect(result.isError == true) + #expect(result.structuredContent == .object([ + "engram_write_receipt": .object([ + "schema_version": .int(1), "tool": .string("remember"), + "write_outcome": .string("not_stored_transaction_not_started"), + "reason": .string("database_busy"), "memory_ids": .array([]), + ]), ])) + // Verify the actual MCP wire shape, including no extra receipt fields. + let wire = try #require(JSONSerialization.jsonObject(with: JSONEncoder().encode(result)) as? [String: Any]) + #expect(Set(wire.keys) == ["content", "structuredContent", "isError"]) + let content = try #require(wire["content"] as? [[String: String]]) + #expect(content == [["type": "text", "text": "Memory was not stored: the database was busy before the write transaction started. Retry on a later turn."]]) + } else { + // Even matching busy text from inside the body is not a no-write + // attestation; the caller still receives an unknown storage error. + await #expect(throws: (any Error).self) { + _ = try await fixture.tools.handle(request) + } } #expect(Date().timeIntervalSince(started) < 5) // New independent readers prove committed state, rather than inspecting diff --git a/Tests/codex_plugin/test_host.py b/Tests/codex_plugin/test_host.py index 755bd4b..e3164a8 100644 --- a/Tests/codex_plugin/test_host.py +++ b/Tests/codex_plugin/test_host.py @@ -333,6 +333,40 @@ def test_known_no_write_conflict_failure_keeps_retry_without_reconciliation(self self.assertEqual(result, 'failed') self.assertNotIn('reconciliation_required', state) + def test_begin_busy_keeps_cursor_and_recovers_on_later_ordinary_event(self): + sid, path, payload, result, state = self.failure_with_audit([ + {'event': 'relay_started'}, {'event': 'tool_call', 'id': 1, 'tool': 'remember'}, + {'event': 'tool_result', 'id': 1, 'tool': 'remember', 'ok': False, + 'forwarded': True, 'memory_ids': [], 'write_outcome': 'not_stored_transaction_not_started', + 'write_outcome_version': 1}, + {'event': 'relay_finished', 'child_reaped': True, 'cleanup_overrun': False}]) + self.assertEqual(result, 'failed') + self.assertNotIn('reconciliation_required', state) + self.assertEqual(state['offset'], self.request(sid)['admission']['frontier_offset']) + self.assertGreater(state['retry_after'], time.time()) + original = F.RUNNER.process_request + def process(root, request, config): + return original(root, request, config, invoke=self.invoke) + with mock.patch.object(F.RUNNER, 'process_request', side_effect=process), \ + mock.patch.object(F.RUNNER, 'spawn_worker', side_effect=F.forbidden): + F.RUNNER.worker(self.root) # Respect backoff without another provider. + paused = self.request(sid) + self.assertEqual(paused['paused_request_id'], paused['request_id']) + self.assertEqual(self.invocations, []) + with mock.patch.object(F.RUNNER.time, 'time', return_value=state['retry_after'] + 1): + F.RUNNER.worker(self.root) # Elapsed time alone does not replay. + self.assertEqual(self.invocations, []) + self.append(path, 'NEXT NATURAL TURN') + self.assertTrue(self.dispatch(payload, turn_id='next-turn')) + F.RUNNER.worker(self.root) + self.assertEqual(len(self.invocations), 1) + self.assertIn('DURABLE NEW TEXT', self.invocations[0][1].text) + self.assertIn('NEXT NATURAL TURN', self.invocations[0][1].text) + recovered = json.loads((self.root / 'sessions' / (sid + '.json')).read_text()) + self.assertEqual(recovered['status'], 'succeeded') + self.assertGreater(recovered['offset'], state['offset']) + self.assertFalse((self.root / 'pending' / (sid + '.json')).exists()) + def test_unrecognized_forwarded_no_ids_response_stays_held(self): _, _, _, result, state = self.failure_with_audit([ {'event': 'relay_started'}, {'event': 'tool_call', 'id': 1, 'tool': 'remember'}, diff --git a/Tests/codex_plugin/test_receipts.py b/Tests/codex_plugin/test_receipts.py index 499538d..dc7060a 100644 --- a/Tests/codex_plugin/test_receipts.py +++ b/Tests/codex_plugin/test_receipts.py @@ -1,5 +1,6 @@ """Proxy budget, privacy and verified receipt regressions with no transport child.""" import importlib +import copy import json from pathlib import Path import tempfile @@ -117,6 +118,101 @@ def test_malformed_no_write_receipts_fail_both_completion_and_reconciliation(sel (path / 'mcp-audit.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in rows)) self.assertIsNotNone(F.RUNNER.failure_reconciliation(path, True)) + def begin_busy(self): + return {'isError': True, 'content': [{'type': 'text', 'text': PROXY.BEGIN_BUSY_TEXT}], + 'structuredContent': {'engram_write_receipt': { + 'schema_version': 1, 'tool': 'remember', + 'write_outcome': 'not_stored_transaction_not_started', + 'reason': 'database_busy', 'memory_ids': []}}} + + def busy_reply(self, number, result=None, **envelope): + return self.policy.server_message({'jsonrpc': '2.0', 'id': number, + 'result': self.begin_busy() if result is None else result, **envelope}) + + def test_begin_busy_is_failed_but_replay_has_no_unverified_write(self): + self.call(1, 'remember', {'content': 'Synthetic'}) + output, _ = self.busy_reply(1) + self.assertIs(output['result']['isError'], True) + self.assertEqual(self.audit.rows[-1], { + 'event': 'tool_result', 'id': 1, 'tool': 'remember', 'ok': False, + 'forwarded': True, 'memory_ids': [], + 'write_outcome': 'not_stored_transaction_not_started', 'write_outcome_version': 1}) + self.assertEqual(self.audit_result(), {'tool_calls': 1, 'write_calls': 0, + 'writes': [], 'tool_errors': 1}) + self.assertIsNone(self.reconciliation([*self.timeout_prefix(), *self.audit.rows])) + # Failure still consumes the finite attempt budget. + _, denied = self.call(2, 'remember', {'content': 'Retry'}) + self.assertTrue(denied['result']['isError']) + + def test_begin_busy_contract_rejects_text_spoofs_and_contradictions(self): + base = self.begin_busy() + candidates = [] + for key in base: + value = copy.deepcopy(base); del value[key]; candidates.append(value) + for replacement in [False, 1, None]: + candidates.append(dict(base, isError=replacement)) + candidates.extend([dict(base, extra=True), dict(base, content=[]), + dict(base, content=[{'type': 'text', 'text': 'Quoted: ' + PROXY.BEGIN_BUSY_TEXT}]), + dict(base, content=[{'type': 'text', 'text': PROXY.BEGIN_BUSY_TEXT, 'extra': True}]), + dict(base, structuredContent={'other': base['structuredContent']})]) + mutations = [('schema_version', True), ('schema_version', 1.0), ('schema_version', 2), + ('tool', 'update'), ('write_outcome', 'unknown'), ('reason', 'other'), + ('memory_ids', [ID]), ('memory_ids', None), ('extra', True)] + for key, value in mutations: + changed = copy.deepcopy(base) + changed['structuredContent']['engram_write_receipt'][key] = value + candidates.append(changed) + for key in base['structuredContent']['engram_write_receipt']: + changed = copy.deepcopy(base); del changed['structuredContent']['engram_write_receipt'][key] + candidates.append(changed) + for candidate in candidates: + with self.subTest(candidate=candidate): + self.audit = MemoryAudit() + self.policy = PROXY.Policy(self.audit, 'codex-session:SOURCE', 3, 1) + self.call(1, 'remember', {'content': 'Synthetic'}) + self.busy_reply(1, candidate) + self.assertNotIn('write_outcome', self.audit.rows[-1]) + self.assertIsNotNone(self.reconciliation([*self.timeout_prefix(), *self.audit.rows])) + self.assertFalse(PROXY.verified_begin_busy_response('update', base)) + self.audit = MemoryAudit(); self.policy = PROXY.Policy(self.audit, 'codex-session:SOURCE', 3, 1) + self.call(1, 'remember', {'content': 'Synthetic'}) + self.busy_reply(1, error={'code': -1, 'message': 'Contradictory error'}) + self.assertNotIn('write_outcome', self.audit.rows[-1]) + self.assertIsNotNone(self.reconciliation([*self.timeout_prefix(), *self.audit.rows])) + + def test_begin_busy_receipt_mutations_remain_uncertain(self): + self.call(1, 'remember', {'content': 'Synthetic'}); self.busy_reply(1) + valid = self.audit.rows[-1] + mutations = [{'write_outcome_version': True}, {'write_outcome_version': 1.0}, + {'write_outcome_version': 2}, {'write_outcome': 'unknown'}, + {'ok': True}, {'ok': 0}, {'forwarded': False}, {'forwarded': 1}, + {'memory_ids': [ID]}, {'memory_ids': None}, {'tool': 'update'}, {'extra': True}] + candidates = [dict(valid, **change) for change in mutations] + candidates.extend({k: v for k, v in valid.items() if k != removed} for removed in valid) + for candidate in candidates: + with self.subTest(candidate=candidate): + self.assertFalse(PROXY.verified_no_write_receipt(candidate)) + self.assertIsNotNone(self.reconciliation([*self.timeout_prefix(), self.audit.rows[0], candidate])) + + def test_safe_begin_failure_cannot_erase_prior_success_or_uncertainty(self): + for prior in (None, {'ok': False, 'memory_ids': []}, {'ok': True, 'memory_ids': [ID]}): + with self.subTest(prior=prior): + self.audit = MemoryAudit(); self.policy = PROXY.Policy(self.audit, 'codex-session:SOURCE', 3, 2) + self.call(2, 'remember', {'content': 'Synthetic'}); self.busy_reply(2) + rows = [*self.timeout_prefix(), {'event': 'tool_call', 'id': 1, 'tool': 'remember'}] + if prior is not None: + rows.append({'event': 'tool_result', 'id': 1, 'tool': 'remember', + 'forwarded': True, **prior}) + rows.extend(self.audit.rows) + gate = self.reconciliation(rows) + self.assertEqual(gate['reason'], 'successful_or_unverified_write') + self.assertEqual(gate['memory_ids'], [ID] if prior and prior['ok'] else []) + + def test_post_timeout_cleanup_cannot_claim_native_begin_receipt(self): + self.call(1, 'remember', {'content': 'Synthetic'}); self.busy_reply(1) + rows = [*self.timeout_prefix(), self.audit.rows[0], {'event': 'request_timeout'}, self.audit.rows[1]] + self.assertEqual(self.reconciliation(rows)['reason'], 'write_status_unknown') + def test_update_receipt_requires_requested_exact_uuid(self): self.call(1, 'update', {'id': ID, 'content': 'Synthetic'}) self.reply(1, f'Updated memory (id: {OTHER})') diff --git a/codex/plugins/engram/.codex-plugin/plugin.json b/codex/plugins/engram/.codex-plugin/plugin.json index 2c1e9b0..cb7e9ff 100644 --- a/codex/plugins/engram/.codex-plugin/plugin.json +++ b/codex/plugins/engram/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "engram", - "version": "0.14.8", + "version": "0.14.9", "description": "Engram memory tools, skills, contextual advice, and session learning for Codex.", "author": { "name": "Engram" diff --git a/codex/plugins/engram/scripts/codex_learner/memory_proxy.py b/codex/plugins/engram/scripts/codex_learner/memory_proxy.py index 78ca44f..de29351 100644 --- a/codex/plugins/engram/scripts/codex_learner/memory_proxy.py +++ b/codex/plugins/engram/scripts/codex_learner/memory_proxy.py @@ -79,6 +79,9 @@ def rpc_error(request_id: Any, message: str) -> dict[str, Any]: # branch in MemoryTools+Core.swift. Match the outer native receipt, never a # phrase or UUID quoted inside remembered content. NO_WRITE_OUTCOME = "not_stored_near_duplicate" +BEGIN_BUSY_OUTCOME = "not_stored_transaction_not_started" +BEGIN_BUSY_VERSION = 1 +BEGIN_BUSY_TEXT = "Memory was not stored: the database was busy before the write transaction started. Retry on a later turn." NEAR_DUPLICATE_PREFIX = "⚠️ Near-duplicate memory detected. The new memory was NOT stored.\n\nExisting similar memories:" NEAR_DUPLICATE_SUFFIX = ('\n\nTo resolve:' '\n - Use `update(id: "UUID", ...)` to modify the existing memory' @@ -102,15 +105,44 @@ def verified_no_write_response(tool: str, result: Any) -> bool: and text.endswith(NEAR_DUPLICATE_SUFFIX)) +def verified_begin_busy_response(tool: str, result: Any) -> bool: + """Accept only native remember's versioned pre-BEGIN no-write contract. + + Text alone, nested quoted content, and any unknown or contradictory field + cannot establish no-write. The native producer checks body entry, not just + an error string; post-entry/commit failures never produce this receipt. + """ + if (tool != "remember" or not isinstance(result, dict) + or set(result) != {"isError", "content", "structuredContent"} + or result["isError"] is not True + or result["content"] != [{"type": "text", "text": BEGIN_BUSY_TEXT}]): + return False + structured = result["structuredContent"] + if not isinstance(structured, dict) or set(structured) != {"engram_write_receipt"}: + return False + receipt = structured["engram_write_receipt"] + return (isinstance(receipt, dict) + and set(receipt) == {"schema_version", "tool", "write_outcome", "reason", "memory_ids"} + and type(receipt["schema_version"]) is int and receipt["schema_version"] == BEGIN_BUSY_VERSION + and receipt["tool"] == "remember" and receipt["write_outcome"] == BEGIN_BUSY_OUTCOME + and receipt["reason"] == "database_busy" + and isinstance(receipt["memory_ids"], list) and receipt["memory_ids"] == []) + + def verified_no_write_receipt(entry: Any) -> bool: """Exact gateway metadata shared by completion and failed-run reconciliation.""" - return (isinstance(entry, dict) - and set(entry) == {"event", "id", "tool", "ok", "memory_ids", "forwarded", "write_outcome"} - and entry.get("event") == "tool_result" and valid_id(entry.get("id")) - and entry.get("tool") == "remember" and entry.get("ok") is True - and entry.get("forwarded") is True - and isinstance(entry.get("memory_ids"), list) and entry["memory_ids"] == [] - and entry.get("write_outcome") == NO_WRITE_OUTCOME) + fields = {"event", "id", "tool", "ok", "memory_ids", "forwarded", "write_outcome"} + if (not isinstance(entry, dict) or entry.get("event") != "tool_result" + or not valid_id(entry.get("id")) or entry.get("tool") != "remember" + or entry.get("forwarded") is not True + or not isinstance(entry.get("memory_ids"), list) or entry["memory_ids"] != []): + return False + if entry.get("write_outcome") == NO_WRITE_OUTCOME: + return set(entry) == fields and entry.get("ok") is True + return (set(entry) == fields | {"write_outcome_version"} + and entry.get("ok") is False and entry.get("write_outcome") == BEGIN_BUSY_OUTCOME + and type(entry.get("write_outcome_version")) is int + and entry["write_outcome_version"] == BEGIN_BUSY_VERSION) def verified_write_ids(tool: str, arguments: dict[str, Any], result: dict[str, Any]) -> list[str]: @@ -177,10 +209,12 @@ def __init__(self, audit: Audit, provenance: str, max_tool_calls: int, max_write self.message_count = 0 self.max_messages = max(64, max_tool_calls * 4 + 8) - def record_result(self, request_id: Any, tool: str, ok: bool, memory_ids: list[str] | None = None, *, forwarded: bool = True, write_outcome: str | None = None) -> None: + def record_result(self, request_id: Any, tool: str, ok: bool, memory_ids: list[str] | None = None, *, forwarded: bool = True, write_outcome: str | None = None, write_outcome_version: int | None = None) -> None: entry = {"event": "tool_result", "id": request_id, "tool": tool, "ok": ok, "memory_ids": memory_ids or [], "forwarded": forwarded} if write_outcome is not None: entry["write_outcome"] = write_outcome + if write_outcome_version is not None: + entry["write_outcome_version"] = write_outcome_version self.audit.write(entry) def deny_tool(self, request_id: Any, tool: str, reason: str) -> tuple[None, dict[str, Any]]: @@ -281,11 +315,13 @@ def server_message(self, message: Any) -> tuple[dict[str, Any] | None, dict[str, tool = pending["tool"] ok = "error" not in message and isinstance(result, dict) and result.get("isError", False) is False no_write = ok and verified_no_write_response(tool, result) + begin_busy = "error" not in message and verified_begin_busy_response(tool, result) memory_ids = verified_write_ids(tool, pending["arguments"], result) if ok and tool in WRITE_TOOLS and not no_write else [] if tool in WRITE_TOOLS and not memory_ids and not no_write: ok = False self.record_result(request_id, tool, ok, memory_ids if ok else [], - write_outcome=NO_WRITE_OUTCOME if no_write else None) + write_outcome=NO_WRITE_OUTCOME if no_write else (BEGIN_BUSY_OUTCOME if begin_busy else None), + write_outcome_version=BEGIN_BUSY_VERSION if begin_busy else None) if not ok and "error" not in message and isinstance(result, dict) and result.get("isError", False) is False: return tool_error(request_id, "write was not verified by an Engram receipt; do not retry blindly"), None return message, None diff --git a/codex/plugins/engram/scripts/codex_learner/runner.py b/codex/plugins/engram/scripts/codex_learner/runner.py index eb47a59..449dc3d 100644 --- a/codex/plugins/engram/scripts/codex_learner/runner.py +++ b/codex/plugins/engram/scripts/codex_learner/runner.py @@ -785,7 +785,8 @@ def unique_fields(pairs): continue if request_timed_out and (event in {"tool_call", "initialize_compat"} or (event == "tool_result" and - (entry.get("ok") is not False or entry.get("forwarded") is not True))): + (entry.get("ok") is not False or entry.get("forwarded") is not True + or "write_outcome" in entry or "write_outcome_version" in entry))): return unknown if event in {"relay_finished", "relay_interrupted", "relay_failed", "relay_cleanup_failed", "initialize_compat"}: continue @@ -811,7 +812,9 @@ def unique_fields(pairs): return unknown if result is not None and "write_outcome" in result: if verified_no_write_receipt(result): - continue # A vetted native conflict completed without storage. + # Vetted native conflict or failed BEGIN; this attempt + # stored nothing. Any earlier risky attempt remains risky. + continue return unknown # Contradictory/unknown outcome metadata is unsafe. if result is not None and result.get("forwarded") is False and result.get("ok") is False: continue # A denied call is proved not to have reached Engram. diff --git a/release-train/README.md b/release-train/README.md index c6d5021..87c9d6b 100644 --- a/release-train/README.md +++ b/release-train/README.md @@ -37,7 +37,7 @@ python3 release-train/release_train.py dispatch \ `dispatch` starts the configured GitHub-hosted release workflow after candidate selection. Ordinary hosted CI and these established hosted release gates are separate from the local native resource queue. Local builds, signing, GUI and model checks still require the existing ROOT allocation. Lattice calls its existing `ci.yml` plus release preflight; LatticeCore reuses its macOS, Linux and C ABI definitions; Engram runs its Linux portable build, existing native tests, app/CLI signing, notarization and appcast generation. The exact `GITHUB_SHA` is the source of every job. Publication rechecks that source and dependency selection. If `main` advances during a run, publication waits for the new candidate instead of silently shipping an old one. -Direct `workflow_dispatch` avoids depending on a tag pushed with `GITHUB_TOKEN` triggering another workflow. Legacy tag pushes still enter the same gates. The newly added dispatch path does not send Slack/email notifications. Existing Engram tag-push notifications retain their existing entry point; the release owner uses dispatch. +Direct `workflow_dispatch` avoids depending on a tag pushed with `GITHUB_TOKEN` triggering another workflow. Legacy tag pushes still enter the same gates. Engram announces successful publication in Slack for both dispatch and tag-push entry points. Its preflight requires the configured webhook, and the notification step checks Slack's acknowledgment. A failed notification is reported separately from the already published release; do not rerun publication to resend it. A successful hosted release uploads `release-receipt.json`, binding the exact source, candidate digest, artifact SHA-256/byte counts and Actions run/attempt. Source-library releases have an empty additional artifact set; GitHub supplies their tagged source archives. App releases require the complete configured signed distribution artifacts. A GitHub source tag used by SwiftPM is checked by exact tag-to-commit resolution; a GitHub Release web page is not required for a dependency's already published source tag. diff --git a/scripts/test_codex_release_workflow.py b/scripts/test_codex_release_workflow.py index 350b960..75ad10b 100644 --- a/scripts/test_codex_release_workflow.py +++ b/scripts/test_codex_release_workflow.py @@ -53,8 +53,21 @@ def test_release_and_linux_commands_enforce_locks_and_xcode_version(self): with tempfile.TemporaryDirectory(prefix='engram-workflow-') as temporary: root = Path(temporary) (root / 'scripts').mkdir() - shutil.copyfile(ROOT / 'scripts/run_native_tests.py', - root / 'scripts/run_native_tests.py') + # This test checks workflow command contracts. Supervision has its + # own tests; a real process inventory can stall on a busy host. + (root / 'scripts/run_native_tests.py').write_text('''import argparse, subprocess +parser = argparse.ArgumentParser() +parser.add_argument('--diagnostics-dir', required=True) +parser.add_argument('--timeout-seconds', type=int, required=True) +parser.add_argument('--silence-seconds', type=int, required=True) +parser.add_argument('command', nargs=argparse.REMAINDER) +args = parser.parse_args() +assert args.diagnostics_dir == 'build/native-test-diagnostics', args +assert args.timeout_seconds == 1800, args +assert args.silence_seconds == 300, args +assert args.command and args.command[0] == '--', args +raise SystemExit(subprocess.run(args.command[1:], check=False).returncode) +''') bin_dir = root / 'bin' bin_dir.mkdir() log = root / 'calls.jsonl' @@ -101,7 +114,7 @@ def test_release_and_linux_commands_enforce_locks_and_xcode_version(self): self.assertTrue(any('clusters_statementBudget' in call for call in tests)) self.assertEqual(tests[0], [ 'swift', 'test', '--force-resolved-versions', '--skip-build', '--filter', - 'EngramTests|EngramMemoryCoreTests|EngramRealityKitTests|PositionVersionTests', + 'EngramTests|EngramMemoryCoreTests|EngramRealityKitTests|PositionVersionTests|LockedSnapshotTests', '--skip', 'PerfTests', '--skip', 'keyBERTKeywordExtraction', '--skip', 'recall_semanticRelevanceOrdering', '--skip', 'recall_connectedMemory_showsEdgeRelation', @@ -118,5 +131,47 @@ def test_native_mcp_gate_includes_persistence_and_existing_cases(self): self.assertEqual(len(cases), 5) + def test_slack_dispatch_announces_publication_and_checks_acknowledgment(self): + workflow = (ROOT / '.github/workflows/release.yml').read_text() + step = workflow.split(' - name: Notify Slack\n', 1)[1].split('\n - ', 1)[0] + self.assertIn("steps.publish.outcome == 'success'", step) + self.assertNotIn("github.event_name", step) + self.assertIn("SWIFTLM SLACK; do", workflow) + with tempfile.TemporaryDirectory(prefix='engram-slack-') as temporary: + root = Path(temporary) + bin_dir = root / 'bin' + bin_dir.mkdir() + fake = '#!' + sys.executable + '\n' + """import json,os,sys +from pathlib import Path +args=sys.argv[1:] +assert '--fail' in args and '--max-time' in args,args +assert '--retry' not in args,args +payload=json.loads(args[args.index('-d')+1]) +Path(os.environ['CALL_LOG']).write_text(json.dumps(payload)) +print(os.environ['FAKE_ACK']) +sys.exit(int(os.environ.get('FAKE_EXIT','0'))) +""" + command = bin_dir / 'curl' + command.write_text(fake) + command.chmod(0o755) + log = root / 'payload.json' + (root / 'CHANGELOG.md').write_text('## [0.14.9]\n### Fixed\n- A "quoted" fix\n## [0.14.8]\n- Old\n') + block = run_block('release.yml', 'Notify Slack').replace('${{ github.repository }}', 'owner/Engram') + env = dict(os.environ, PATH=f'{bin_dir}:/usr/bin:/bin', CALL_LOG=str(log), + RELEASE_TAG='v0.14.9', SLACK_WEBHOOK_URL='https://example.invalid/webhook') + for ack, exit_code, expected in [('ok', '0', 0), ('invalid_payload', '0', 1), ('ok', '22', 22)]: + with self.subTest(ack=ack, exit_code=exit_code): + result = subprocess.run(['/bin/bash', '-e', '-c', block], cwd=root, + env=dict(env, FAKE_ACK=ack, FAKE_EXIT=exit_code), + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, expected, result.stdout + result.stderr) + payload = json.loads(log.read_text()) + self.assertIn('v0.14.9', payload['blocks'][0]['text']['text']) + self.assertEqual(payload['blocks'][1]['accessory']['url'], + 'https://github.com/owner/Engram/releases/tag/v0.14.9') + self.assertIn('A "quoted" fix', payload['blocks'][1]['text']['text']) + self.assertNotIn('Old', payload['blocks'][1]['text']['text']) + + if __name__ == '__main__': unittest.main() From 7c3c3a1401f3ab48f23a3e04b5eec459e1fbbb2b Mon Sep 17 00:00:00 2001 From: jsflax Date: Wed, 23 Sep 2026 19:45:01 -0400 Subject: [PATCH 7/8] Respect the explicit native test fixture root --- Tests/EngramTests/ClaudeMCPRegistrationTests.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tests/EngramTests/ClaudeMCPRegistrationTests.swift b/Tests/EngramTests/ClaudeMCPRegistrationTests.swift index 457f154..6ad61d0 100644 --- a/Tests/EngramTests/ClaudeMCPRegistrationTests.swift +++ b/Tests/EngramTests/ClaudeMCPRegistrationTests.swift @@ -10,7 +10,10 @@ private struct MCPRegistrationFixture { let config: URL init() throws { - home = FileManager.default.temporaryDirectory.appendingPathComponent("engram-mcp-\(UUID().uuidString)") + let root = ProcessInfo.processInfo.environment["ENGRAM_ADOPTION_TEST_ROOT"] + .map { URL(fileURLWithPath: $0, isDirectory: true) } + ?? FileManager.default.temporaryDirectory + home = root.appendingPathComponent("engram-mcp-\(UUID().uuidString)") memory = home.appendingPathComponent(".claude/bin/memory") claude = home.appendingPathComponent(".local/bin/claude") config = home.appendingPathComponent(".claude.json") From e7321655fa43969e96c29ef263d1fd6054f3d0c2 Mon Sep 17 00:00:00 2001 From: jsflax Date: Wed, 23 Sep 2026 19:48:35 -0400 Subject: [PATCH 8/8] Keep registration fixture writes inside their sandbox --- Tests/EngramTests/ClaudeMCPRegistrationTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/EngramTests/ClaudeMCPRegistrationTests.swift b/Tests/EngramTests/ClaudeMCPRegistrationTests.swift index 6ad61d0..aba10bc 100644 --- a/Tests/EngramTests/ClaudeMCPRegistrationTests.swift +++ b/Tests/EngramTests/ClaudeMCPRegistrationTests.swift @@ -25,7 +25,7 @@ private struct MCPRegistrationFixture { func cleanup() { try? FileManager.default.removeItem(at: home) } func writeExecutable(_ url: URL, _ contents: String) throws { - try contents.write(to: url, atomically: true, encoding: .utf8) + try contents.write(to: url, atomically: false, encoding: .utf8) try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: url.path) } @@ -102,7 +102,7 @@ func claudeMCPRegistrationPreservesMalformedConfiguration() throws { let fixture = try MCPRegistrationFixture(); defer { fixture.cleanup() } try fixture.successfulClaude() for contents in ["{broken", "[]", "{\"mcpServers\":null}"] { - try contents.write(to: fixture.config, atomically: true, encoding: .utf8) + try contents.write(to: fixture.config, atomically: false, encoding: .utf8) #expect(fixture.register() == .invalidConfiguration) #expect(try String(contentsOf: fixture.config, encoding: .utf8) == contents) }