fix(evals): end the Codex skill listing at the closing tag

evals/promises/harnesses/codex.py

@@ -32,6 +32,7 @@
 PERSONA_LINE = re.compile(r"persona: ([\w-]+)")
 REPLY_PERSONA_LINE = re.compile(r"^persona: ([\w-]+)[ \t]*$", re.M)
 AVAILABLE_SKILLS = "### Available skills"
+LISTING_END = re.compile(r"</skills_instructions>|\n#")
 SKILL_INJECTION = re.compile(r"^<skill>\n<name>([^<]+)</name>\n<path>([^<]+)</path>")
 
 
@@ -269,7 +270,7 @@ def add(event):
                 text = text_of(payload.get("content"))
                 hit = SKILL_INJECTION.match(text)
                 if payload.get("role") == "developer" and listed is None and AVAILABLE_SKILLS in text:
-                    listed = live.listed_skills(text.split(AVAILABLE_SKILLS, 1)[1].split("\n#", 1)[0])
+                    listed = live.listed_skills(LISTING_END.split(text.split(AVAILABLE_SKILLS, 1)[1], 1)[0])
                 if payload.get("role") == "user" and hit:
                     injected.append({"name": hit.group(1), "path": hit.group(2), "turn": turn if prompts is not None else None})
                 if payload.get("role") == "user" and upcoming and text.strip() == upcoming[0][1].strip():

evals/promises/test_codex.py

@@ -34,16 +34,36 @@ def item(payload):
 
 
 class SkillListing(unittest.TestCase):
-    def test_the_lead_rollout_listing_reaches_the_harvest_without_its_file_paths(self):
-        listing = ("## Skills\n### Skill roots\n- `r1` = `/w/.agents/skills`\n### Available skills\n"
-                   "- deslop: Remove AI slop. (file: r1/deslop/SKILL.md)\n- how: Use for how X works. (file: r1/how/SKILL.md)\n"
-                   "</skills_instructions>")
+    ENTRIES = ("## Skills\n### Skill roots\n- `r1` = `/w/.agents/skills`\n### Available skills\n"
+               "- deslop: Remove AI slop. (file: r1/deslop/SKILL.md)\n- how: Use for how X works. (file: r1/how/SKILL.md)\n")
+    LISTED = {"deslop": "Remove AI slop.", "how": "Use for how X works."}
+
+    def harvest(self, developer_text):
         rows = [{"type": "session_meta", "payload": {"cwd": "/w", "id": "s"}},
-                {"type": "response_item", "payload": {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": listing}]}}]
-        data = "".join(json.dumps(r) + "\n" for r in rows).encode()
-        self.assertEqual(codex.harvest_rollout(data)["listed"], {"deslop": "Remove AI slop.", "how": "Use for how X works."})
+                {"type": "response_item", "payload": {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": developer_text}]}}]
+        return "".join(json.dumps(r) + "\n" for r in rows).encode()
+
+    def test_the_lead_rollout_listing_reaches_the_harvest_without_its_file_paths(self):
+        data = self.harvest(self.ENTRIES + "</skills_instructions>")
+        self.assertEqual(codex.harvest_rollout(data)["listed"], self.LISTED)
         self.assertEqual(codex.harvest_rollout(data[:data.index(b"\n") + 1])["listed"], None)
 
+    def test_a_how_to_use_section_before_the_closing_tag_is_not_part_of_the_listing(self):
+        text = self.ENTRIES + "### How to use skills\n- Discovery: the list above is the skills available.\n</skills_instructions>"
+        self.assertEqual(codex.harvest_rollout(self.harvest(text))["listed"], self.LISTED)
+
+    def test_the_listing_ends_at_the_closing_tag_when_the_next_heading_is_further_on(self):
+        after = ("<permissions instructions>\nFilesystem sandboxing defines which files can be read or written. "
+                 "`sandbox_mode` is `workspace-write`: The sandbox permits reading files, and editing files in `cwd` and `writable_roots`. "
+                 "Editing files in other directories requires approval. Network access is restricted.\n"
+                 "- network: restricted\n"
+                 "Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.\n"
+                 " The writable roots are `/w/tmp`, `/w/relay`, `/w/relay/.git`, `/private/tmp`.\n</permissions instructions>"
+                 "<collaboration_mode># Collaboration Mode: Default\n\nYou are now in Default mode.\n\n"
+                 "## request_user_input availability\n\nUse the `request_user_input` tool only when it is listed.")
+        text = self.ENTRIES + "</skills_instructions>" + after
+        self.assertEqual(codex.harvest_rollout(self.harvest(text))["listed"], self.LISTED)
+
 
 class Compaction(unittest.TestCase):
     def setUp(self):