Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 43 additions & 1 deletion loopx/codex_app_thread_activity.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,38 @@ def observe_codex_thread_execution_identity(
return {"status": "unavailable", "reason": fallback}


def _rollout_stamp(stat: os.stat_result) -> tuple[int, int, int, int]:
return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns


def _confirm_windows_rollout(
path: Path, cursor: RolloutCursor, source: os.stat_result,
result: HostThreadActivity, reader: Reader,
) -> bool:
if not reader.remaining or result.state is HostThreadState.UNKNOWN:
return False
# A fresh descriptor and cursor verify the header and independently choose
# the marker and latest event, including when the first read used a
# retained forward cursor. The repeat uses only the unused read budget.
expected = _rollout_stamp(source)
with open_rollout(path) as handle:
before = os.fstat(handle.fileno())
if _rollout_stamp(before) != expected:
return False
fresh = RolloutCursor(cursor.binding, cursor.thread_id)
confirmation = fresh.observe(
Reader(handle, reader.remaining, chunk_bytes=reader.chunk_bytes), before,
)
after = os.fstat(handle.fileno())
return (
confirmation == result
and _rollout_stamp(after) == expected
and after.st_ctime_ns == before.st_ctime_ns
and Path(os.path.realpath(path)) == path
and _rollout_stamp(path.stat()) == expected
)


def _rollout_activity(
rollout_path: Any, home: Path, *, thread_id: str, cache_root: Path | None = None,
) -> HostThreadActivity:
Expand Down Expand Up @@ -246,8 +278,18 @@ def _rollout_activity(
if Path(os.path.realpath(path)) != path or (end.st_dev, end.st_ino) != (named.st_dev, named.st_ino) or end.st_size < stat.st_size:
_rollout_cache.pop(binding, None)
return HostThreadActivity.unknown(HostThreadUnknownReason.STORE_UNAVAILABLE)
if end.st_size == stat.st_size and (end.st_mtime_ns != stat.st_mtime_ns or end.st_ctime_ns != stat.st_ctime_ns):
if end.st_size == stat.st_size and end.st_mtime_ns != stat.st_mtime_ns:
_rollout_cache.pop(binding, None)
return HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED)
if end.st_size == stat.st_size and end.st_ctime_ns != stat.st_ctime_ns:
# Don't persist the first cursor across ambiguous metadata.
_rollout_cache.pop(binding, None)
if (
os.name == "nt" and stat.st_dev and stat.st_ino
and _rollout_stamp(stat) == _rollout_stamp(end)
and _confirm_windows_rollout(path, cursor, end, result, reader)
):
return result
return HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED)
cursor.save(cache_root)
# An append may contain a newer end/cancel. Do not publish an old
Expand Down
180 changes: 180 additions & 0 deletions tests/test_host_thread_activity.py
Original file line number Diff line number Diff line change
Expand Up @@ -1153,3 +1153,183 @@ def observe(thread_ids: Any) -> dict[str, HostThreadActivity]:
expectations=lambda scopes: pytest.fail("no observed lane may ask for a cadence"),
)
assert unattached == expected


@pytest.mark.parametrize("unavailable_identity", [None, "st_dev", "st_ino"])
@pytest.mark.parametrize("platform", ["nt", "posix"])
@pytest.mark.parametrize("field", ["st_mtime_ns", "st_ctime_ns"])
def test_same_size_read_fence_metadata_policy(home, monkeypatch, platform, field, unavailable_identity):
rollout = home.thread("t", [_event(T2, "task_complete")])
rollout_stat = rollout.stat()
assert rollout_stat.st_dev and rollout_stat.st_ino
original_fstat = os.fstat
original_stat = Path.stat
descriptor_observations = 0

def partial_identity(observed):
if unavailable_identity is None or (observed.st_dev, observed.st_ino) != (
rollout_stat.st_dev, rollout_stat.st_ino
):
return observed
# Unavailable volume/file indexes are injected at every rollout stat.
fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")}
fields[unavailable_identity] = 0
return SimpleNamespace(**fields)

def path_stat(self, *args, **kwargs):
return partial_identity(original_stat(self, *args, **kwargs))

def descriptor_stat(fd):
nonlocal descriptor_observations
observed = original_fstat(fd)
if (observed.st_dev, observed.st_ino) != (rollout_stat.st_dev, rollout_stat.st_ino):
return observed
descriptor_observations += 1
observed = partial_identity(observed)
if descriptor_observations != 2:
return observed
fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")}
fields[field] += 1
return SimpleNamespace(**fields)

monkeypatch.setattr(Path, "stat", path_stat)
monkeypatch.setattr(codex_activity, "os", SimpleNamespace(**{
**vars(os), "name": platform, "fstat": descriptor_stat,
}))
result = observe_codex_threads(["t"], homes=[home.root])["t"]
if platform == "nt" and field == "st_ctime_ns" and unavailable_identity is None:
assert result.state is HostThreadState.IDLE
assert result.last_turn_ended_at == T2
assert descriptor_observations == 4
else:
assert result == HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED)
assert descriptor_observations == 2


@pytest.mark.parametrize("warm_cursor", [False, True], ids=["cold", "forward"])
@pytest.mark.parametrize("change", ["none", "marker", "latest", "append"])
def test_windows_ctime_skew_checks_current_rollout_records(home, monkeypatch, warm_cursor, change):
records = [_event(T1, "task_complete"), _item(T2)]
if warm_cursor:
path = home.thread("t", [_event(T0, "task_started")])
assert _observe(home, "t")["t"].state is HostThreadState.TURN_OPEN
with path.open("ab") as handle:
handle.write("".join(json.dumps(row) + "\n" for row in records).encode())
else:
path = home.thread("t", records)
initial = path.stat()
old_bytes = path.read_bytes()
old_identity = (initial.st_dev, initial.st_ino)
observed_stats = 0

def fstat(fd):
nonlocal observed_stats
observed = os.fstat(fd)
if (observed.st_dev, observed.st_ino) != old_identity:
return observed
observed_stats += 1
if observed_stats != 2:
return observed
if change == "append":
with path.open("ab") as handle:
handle.write((json.dumps(_event(T3, "task_started")) + "\n").encode())
return os.fstat(fd)
if change in {"marker", "latest"}:
new_bytes = (
old_bytes.replace(b'"task_complete"', b'"task_started" ')
if change == "marker"
else old_bytes.replace(T2.encode(), T3.encode())
)
assert old_bytes != new_bytes and len(old_bytes) == len(new_bytes)
path.write_bytes(new_bytes)
os.utime(path, ns=(initial.st_atime_ns, initial.st_mtime_ns))
fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")}
fields["st_ctime_ns"] = initial.st_ctime_ns + 1
return SimpleNamespace(**fields)

monkeypatch.setattr(codex_activity, "os", SimpleNamespace(**{**vars(os), "name": "nt", "fstat": fstat}))
result = _observe(home, "t")["t"]
if change == "none":
assert result == HostThreadActivity(
state=HostThreadState.IDLE, last_turn_ended_at=T1, last_event_at=T2,
)
elif change == "append":
assert result == HostThreadActivity.unknown(HostThreadUnknownReason.NO_TURN_MARKER)
assert _observe(home, "t")["t"].turn_started_at == T3
else:
assert result == HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED)
refreshed = _observe(home, "t")["t"]
if change == "marker":
assert refreshed == HostThreadActivity(
state=HostThreadState.TURN_OPEN, turn_started_at=T1, last_event_at=T2,
)
else:
assert refreshed == HostThreadActivity(
state=HostThreadState.IDLE, last_turn_ended_at=T1, last_event_at=T3,
)


def test_windows_ctime_skew_does_not_exceed_remaining_read_budget(home, monkeypatch):
path = home.thread("t", [_event(T2, "task_complete")])
total_bytes = 0
read = codex_activity.Reader.read

def count(reader, offset, count):
nonlocal total_bytes
data = read(reader, offset, count)
total_bytes += len(data)
return data

monkeypatch.setattr(codex_activity.Reader, "read", count)
assert _observe(home, "t")["t"].state is HostThreadState.IDLE
budget = total_bytes
monkeypatch.setattr(codex_activity, "_OBSERVATION_READ_BYTES", budget)
codex_activity._rollout_cache.clear()
total_bytes = observations = 0
original = path.stat()

def fstat(fd):
nonlocal observations
observed = os.fstat(fd)
if (observed.st_dev, observed.st_ino) != (original.st_dev, original.st_ino):
return observed
observations += 1
if observations != 2:
return observed
fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")}
fields["st_ctime_ns"] += 1
return SimpleNamespace(**fields)

monkeypatch.setattr(codex_activity, "os", SimpleNamespace(**{**vars(os), "name": "nt", "fstat": fstat}))
result = _observe(home, "t")["t"]
assert result == HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED)
assert total_bytes == budget


@pytest.mark.parametrize(("sample", "field"), [
(3, "st_ino"),
(4, "st_mtime_ns"),
(4, "st_ctime_ns"),
])
def test_windows_ctime_skew_requires_a_stable_second_read(home, monkeypatch, sample, field):
path = home.thread("t", [_event(T2, "task_complete")])
original = path.stat()
sampled_descriptors = []

def fstat(fd):
observed = os.fstat(fd)
if (observed.st_dev, observed.st_ino) != (original.st_dev, original.st_ino):
return observed
sampled_descriptors.append(fd)
sample_number = len(sampled_descriptors)
if sample_number not in {2, sample}:
return observed
fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")}
fields["st_ctime_ns" if sample_number == 2 else field] += 1
return SimpleNamespace(**fields)

monkeypatch.setattr(codex_activity, "os", SimpleNamespace(**{**vars(os), "name": "nt", "fstat": fstat}))
actual = _observe(home, "t")["t"]
assert actual == HostThreadActivity.unknown(HostThreadUnknownReason.RECORD_UNRECOGNIZED)
assert len(sampled_descriptors) == sample
assert sampled_descriptors[0] != sampled_descriptors[2]
Loading