From 13259b2e16e090a719796b3abaf1ab06bee13bce Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:52:07 -0600 Subject: [PATCH] fix(evals): end the Codex skill listing at the closing tag The listing parser cut at the next heading only. Real Codex rollouts end the listing at , and the next heading comes 790 to 1,060 characters later, so any bullet line in the permissions block between them would have been graded as an installed skill. Cut at whichever comes first. Restores the How to use skills heading fixture that 253a46e1 swapped out, and adds one with the permissions text that follows the closing tag. Closes #195. --- evals/promises/harnesses/codex.py | 3 ++- evals/promises/test_codex.py | 34 ++++++++++++++++++++++++------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/evals/promises/harnesses/codex.py b/evals/promises/harnesses/codex.py index b68bc7ab..d21f662f 100644 --- a/evals/promises/harnesses/codex.py +++ b/evals/promises/harnesses/codex.py @@ -32,6 +32,7 @@ PERSONA_LINE = re.compile(r"persona: ([\w-]+)") REPLY_PERSONA_LINE = re.compile(r"^persona: ([\w-]+)[ \t]*$", re.M) AVAILABLE_SKILLS = "### Available skills" +LISTING_END = re.compile(r"|\n#") SKILL_INJECTION = re.compile(r"^\n([^<]+)\n([^<]+)") @@ -269,7 +270,7 @@ def add(event): text = text_of(payload.get("content")) hit = SKILL_INJECTION.match(text) if payload.get("role") == "developer" and listed is None and AVAILABLE_SKILLS in text: - listed = live.listed_skills(text.split(AVAILABLE_SKILLS, 1)[1].split("\n#", 1)[0]) + listed = live.listed_skills(LISTING_END.split(text.split(AVAILABLE_SKILLS, 1)[1], 1)[0]) if payload.get("role") == "user" and hit: injected.append({"name": hit.group(1), "path": hit.group(2), "turn": turn if prompts is not None else None}) if payload.get("role") == "user" and upcoming and text.strip() == upcoming[0][1].strip(): diff --git a/evals/promises/test_codex.py b/evals/promises/test_codex.py index effe1ac5..21993216 100644 --- a/evals/promises/test_codex.py +++ b/evals/promises/test_codex.py @@ -34,16 +34,36 @@ def item(payload): class SkillListing(unittest.TestCase): - def test_the_lead_rollout_listing_reaches_the_harvest_without_its_file_paths(self): - listing = ("## Skills\n### Skill roots\n- `r1` = `/w/.agents/skills`\n### Available skills\n" - "- deslop: Remove AI slop. (file: r1/deslop/SKILL.md)\n- how: Use for how X works. (file: r1/how/SKILL.md)\n" - "") + ENTRIES = ("## Skills\n### Skill roots\n- `r1` = `/w/.agents/skills`\n### Available skills\n" + "- deslop: Remove AI slop. (file: r1/deslop/SKILL.md)\n- how: Use for how X works. (file: r1/how/SKILL.md)\n") + LISTED = {"deslop": "Remove AI slop.", "how": "Use for how X works."} + + def harvest(self, developer_text): rows = [{"type": "session_meta", "payload": {"cwd": "/w", "id": "s"}}, - {"type": "response_item", "payload": {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": listing}]}}] - data = "".join(json.dumps(r) + "\n" for r in rows).encode() - self.assertEqual(codex.harvest_rollout(data)["listed"], {"deslop": "Remove AI slop.", "how": "Use for how X works."}) + {"type": "response_item", "payload": {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": developer_text}]}}] + return "".join(json.dumps(r) + "\n" for r in rows).encode() + + def test_the_lead_rollout_listing_reaches_the_harvest_without_its_file_paths(self): + data = self.harvest(self.ENTRIES + "") + self.assertEqual(codex.harvest_rollout(data)["listed"], self.LISTED) self.assertEqual(codex.harvest_rollout(data[:data.index(b"\n") + 1])["listed"], None) + def test_a_how_to_use_section_before_the_closing_tag_is_not_part_of_the_listing(self): + text = self.ENTRIES + "### How to use skills\n- Discovery: the list above is the skills available.\n" + self.assertEqual(codex.harvest_rollout(self.harvest(text))["listed"], self.LISTED) + + def test_the_listing_ends_at_the_closing_tag_when_the_next_heading_is_further_on(self): + after = ("\nFilesystem sandboxing defines which files can be read or written. " + "`sandbox_mode` is `workspace-write`: The sandbox permits reading files, and editing files in `cwd` and `writable_roots`. " + "Editing files in other directories requires approval. Network access is restricted.\n" + "- network: restricted\n" + "Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.\n" + " The writable roots are `/w/tmp`, `/w/relay`, `/w/relay/.git`, `/private/tmp`.\n" + "# Collaboration Mode: Default\n\nYou are now in Default mode.\n\n" + "## request_user_input availability\n\nUse the `request_user_input` tool only when it is listed.") + text = self.ENTRIES + "" + after + self.assertEqual(codex.harvest_rollout(self.harvest(text))["listed"], self.LISTED) + class Compaction(unittest.TestCase): def setUp(self):