diff --git a/loopx/codex_app_thread_activity.py b/loopx/codex_app_thread_activity.py index c81cfa11c4..fba46e8b4e 100644 --- a/loopx/codex_app_thread_activity.py +++ b/loopx/codex_app_thread_activity.py @@ -207,6 +207,38 @@ def observe_codex_thread_execution_identity( return {"status": "unavailable", "reason": fallback} +def _rollout_stamp(stat: os.stat_result) -> tuple[int, int, int, int]: + return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns + + +def _confirm_windows_rollout( + path: Path, cursor: RolloutCursor, source: os.stat_result, + result: HostThreadActivity, reader: Reader, +) -> bool: + if not reader.remaining or result.state is HostThreadState.UNKNOWN: + return False + # A fresh descriptor and cursor verify the header and independently choose + # the marker and latest event, including when the first read used a + # retained forward cursor. The repeat uses only the unused read budget. + expected = _rollout_stamp(source) + with open_rollout(path) as handle: + before = os.fstat(handle.fileno()) + if _rollout_stamp(before) != expected: + return False + fresh = RolloutCursor(cursor.binding, cursor.thread_id) + confirmation = fresh.observe( + Reader(handle, reader.remaining, chunk_bytes=reader.chunk_bytes), before, + ) + after = os.fstat(handle.fileno()) + return ( + confirmation == result + and _rollout_stamp(after) == expected + and after.st_ctime_ns == before.st_ctime_ns + and Path(os.path.realpath(path)) == path + and _rollout_stamp(path.stat()) == expected + ) + + def _rollout_activity( rollout_path: Any, home: Path, *, thread_id: str, cache_root: Path | None = None, ) -> HostThreadActivity: @@ -246,8 +278,18 @@ def _rollout_activity( if Path(os.path.realpath(path)) != path or (end.st_dev, end.st_ino) != (named.st_dev, named.st_ino) or end.st_size < stat.st_size: _rollout_cache.pop(binding, None) return HostThreadActivity.unknown(HostThreadUnknownReason.STORE_UNAVAILABLE) - if end.st_size == stat.st_size and (end.st_mtime_ns != stat.st_mtime_ns or end.st_ctime_ns != stat.st_ctime_ns): + if end.st_size == stat.st_size and end.st_mtime_ns != stat.st_mtime_ns: + _rollout_cache.pop(binding, None) + return HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED) + if end.st_size == stat.st_size and end.st_ctime_ns != stat.st_ctime_ns: + # Don't persist the first cursor across ambiguous metadata. _rollout_cache.pop(binding, None) + if ( + os.name == "nt" and stat.st_dev and stat.st_ino + and _rollout_stamp(stat) == _rollout_stamp(end) + and _confirm_windows_rollout(path, cursor, end, result, reader) + ): + return result return HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED) cursor.save(cache_root) # An append may contain a newer end/cancel. Do not publish an old diff --git a/tests/test_host_thread_activity.py b/tests/test_host_thread_activity.py index 620ebc792b..f9225abe81 100644 --- a/tests/test_host_thread_activity.py +++ b/tests/test_host_thread_activity.py @@ -1153,3 +1153,183 @@ def observe(thread_ids: Any) -> dict[str, HostThreadActivity]: expectations=lambda scopes: pytest.fail("no observed lane may ask for a cadence"), ) assert unattached == expected + + +@pytest.mark.parametrize("unavailable_identity", [None, "st_dev", "st_ino"]) +@pytest.mark.parametrize("platform", ["nt", "posix"]) +@pytest.mark.parametrize("field", ["st_mtime_ns", "st_ctime_ns"]) +def test_same_size_read_fence_metadata_policy(home, monkeypatch, platform, field, unavailable_identity): + rollout = home.thread("t", [_event(T2, "task_complete")]) + rollout_stat = rollout.stat() + assert rollout_stat.st_dev and rollout_stat.st_ino + original_fstat = os.fstat + original_stat = Path.stat + descriptor_observations = 0 + + def partial_identity(observed): + if unavailable_identity is None or (observed.st_dev, observed.st_ino) != ( + rollout_stat.st_dev, rollout_stat.st_ino + ): + return observed + # Unavailable volume/file indexes are injected at every rollout stat. + fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")} + fields[unavailable_identity] = 0 + return SimpleNamespace(**fields) + + def path_stat(self, *args, **kwargs): + return partial_identity(original_stat(self, *args, **kwargs)) + + def descriptor_stat(fd): + nonlocal descriptor_observations + observed = original_fstat(fd) + if (observed.st_dev, observed.st_ino) != (rollout_stat.st_dev, rollout_stat.st_ino): + return observed + descriptor_observations += 1 + observed = partial_identity(observed) + if descriptor_observations != 2: + return observed + fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")} + fields[field] += 1 + return SimpleNamespace(**fields) + + monkeypatch.setattr(Path, "stat", path_stat) + monkeypatch.setattr(codex_activity, "os", SimpleNamespace(**{ + **vars(os), "name": platform, "fstat": descriptor_stat, + })) + result = observe_codex_threads(["t"], homes=[home.root])["t"] + if platform == "nt" and field == "st_ctime_ns" and unavailable_identity is None: + assert result.state is HostThreadState.IDLE + assert result.last_turn_ended_at == T2 + assert descriptor_observations == 4 + else: + assert result == HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED) + assert descriptor_observations == 2 + + +@pytest.mark.parametrize("warm_cursor", [False, True], ids=["cold", "forward"]) +@pytest.mark.parametrize("change", ["none", "marker", "latest", "append"]) +def test_windows_ctime_skew_checks_current_rollout_records(home, monkeypatch, warm_cursor, change): + records = [_event(T1, "task_complete"), _item(T2)] + if warm_cursor: + path = home.thread("t", [_event(T0, "task_started")]) + assert _observe(home, "t")["t"].state is HostThreadState.TURN_OPEN + with path.open("ab") as handle: + handle.write("".join(json.dumps(row) + "\n" for row in records).encode()) + else: + path = home.thread("t", records) + initial = path.stat() + old_bytes = path.read_bytes() + old_identity = (initial.st_dev, initial.st_ino) + observed_stats = 0 + + def fstat(fd): + nonlocal observed_stats + observed = os.fstat(fd) + if (observed.st_dev, observed.st_ino) != old_identity: + return observed + observed_stats += 1 + if observed_stats != 2: + return observed + if change == "append": + with path.open("ab") as handle: + handle.write((json.dumps(_event(T3, "task_started")) + "\n").encode()) + return os.fstat(fd) + if change in {"marker", "latest"}: + new_bytes = ( + old_bytes.replace(b'"task_complete"', b'"task_started" ') + if change == "marker" + else old_bytes.replace(T2.encode(), T3.encode()) + ) + assert old_bytes != new_bytes and len(old_bytes) == len(new_bytes) + path.write_bytes(new_bytes) + os.utime(path, ns=(initial.st_atime_ns, initial.st_mtime_ns)) + fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")} + fields["st_ctime_ns"] = initial.st_ctime_ns + 1 + return SimpleNamespace(**fields) + + monkeypatch.setattr(codex_activity, "os", SimpleNamespace(**{**vars(os), "name": "nt", "fstat": fstat})) + result = _observe(home, "t")["t"] + if change == "none": + assert result == HostThreadActivity( + state=HostThreadState.IDLE, last_turn_ended_at=T1, last_event_at=T2, + ) + elif change == "append": + assert result == HostThreadActivity.unknown(HostThreadUnknownReason.NO_TURN_MARKER) + assert _observe(home, "t")["t"].turn_started_at == T3 + else: + assert result == HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED) + refreshed = _observe(home, "t")["t"] + if change == "marker": + assert refreshed == HostThreadActivity( + state=HostThreadState.TURN_OPEN, turn_started_at=T1, last_event_at=T2, + ) + else: + assert refreshed == HostThreadActivity( + state=HostThreadState.IDLE, last_turn_ended_at=T1, last_event_at=T3, + ) + + +def test_windows_ctime_skew_does_not_exceed_remaining_read_budget(home, monkeypatch): + path = home.thread("t", [_event(T2, "task_complete")]) + total_bytes = 0 + read = codex_activity.Reader.read + + def count(reader, offset, count): + nonlocal total_bytes + data = read(reader, offset, count) + total_bytes += len(data) + return data + + monkeypatch.setattr(codex_activity.Reader, "read", count) + assert _observe(home, "t")["t"].state is HostThreadState.IDLE + budget = total_bytes + monkeypatch.setattr(codex_activity, "_OBSERVATION_READ_BYTES", budget) + codex_activity._rollout_cache.clear() + total_bytes = observations = 0 + original = path.stat() + + def fstat(fd): + nonlocal observations + observed = os.fstat(fd) + if (observed.st_dev, observed.st_ino) != (original.st_dev, original.st_ino): + return observed + observations += 1 + if observations != 2: + return observed + fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")} + fields["st_ctime_ns"] += 1 + return SimpleNamespace(**fields) + + monkeypatch.setattr(codex_activity, "os", SimpleNamespace(**{**vars(os), "name": "nt", "fstat": fstat})) + result = _observe(home, "t")["t"] + assert result == HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED) + assert total_bytes == budget + + +@pytest.mark.parametrize(("sample", "field"), [ + (3, "st_ino"), + (4, "st_mtime_ns"), + (4, "st_ctime_ns"), +]) +def test_windows_ctime_skew_requires_a_stable_second_read(home, monkeypatch, sample, field): + path = home.thread("t", [_event(T2, "task_complete")]) + original = path.stat() + sampled_descriptors = [] + + def fstat(fd): + observed = os.fstat(fd) + if (observed.st_dev, observed.st_ino) != (original.st_dev, original.st_ino): + return observed + sampled_descriptors.append(fd) + sample_number = len(sampled_descriptors) + if sample_number not in {2, sample}: + return observed + fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")} + fields["st_ctime_ns" if sample_number == 2 else field] += 1 + return SimpleNamespace(**fields) + + monkeypatch.setattr(codex_activity, "os", SimpleNamespace(**{**vars(os), "name": "nt", "fstat": fstat})) + actual = _observe(home, "t")["t"] + assert actual == HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED) + assert len(sampled_descriptors) == sample + assert sampled_descriptors[0] != sampled_descriptors[2]