diff --git a/evals/promises/harnesses/codex.py b/evals/promises/harnesses/codex.py index b68bc7ab..d21f662f 100644 --- a/evals/promises/harnesses/codex.py +++ b/evals/promises/harnesses/codex.py @@ -32,6 +32,7 @@ PERSONA_LINE = re.compile(r"persona: ([\w-]+)") REPLY_PERSONA_LINE = re.compile(r"^persona: ([\w-]+)[ \t]*$", re.M) AVAILABLE_SKILLS = "### Available skills" +LISTING_END = re.compile(r"|\n#") SKILL_INJECTION = re.compile(r"^\n([^<]+)\n([^<]+)") @@ -269,7 +270,7 @@ def add(event): text = text_of(payload.get("content")) hit = SKILL_INJECTION.match(text) if payload.get("role") == "developer" and listed is None and AVAILABLE_SKILLS in text: - listed = live.listed_skills(text.split(AVAILABLE_SKILLS, 1)[1].split("\n#", 1)[0]) + listed = live.listed_skills(LISTING_END.split(text.split(AVAILABLE_SKILLS, 1)[1], 1)[0]) if payload.get("role") == "user" and hit: injected.append({"name": hit.group(1), "path": hit.group(2), "turn": turn if prompts is not None else None}) if payload.get("role") == "user" and upcoming and text.strip() == upcoming[0][1].strip(): diff --git a/evals/promises/test_codex.py b/evals/promises/test_codex.py index effe1ac5..21993216 100644 --- a/evals/promises/test_codex.py +++ b/evals/promises/test_codex.py @@ -34,16 +34,36 @@ def item(payload): class SkillListing(unittest.TestCase): - def test_the_lead_rollout_listing_reaches_the_harvest_without_its_file_paths(self): - listing = ("## Skills\n### Skill roots\n- `r1` = `/w/.agents/skills`\n### Available skills\n" - "- deslop: Remove AI slop. (file: r1/deslop/SKILL.md)\n- how: Use for how X works. (file: r1/how/SKILL.md)\n" - "") + ENTRIES = ("## Skills\n### Skill roots\n- `r1` = `/w/.agents/skills`\n### Available skills\n" + "- deslop: Remove AI slop. (file: r1/deslop/SKILL.md)\n- how: Use for how X works. (file: r1/how/SKILL.md)\n") + LISTED = {"deslop": "Remove AI slop.", "how": "Use for how X works."} + + def harvest(self, developer_text): rows = [{"type": "session_meta", "payload": {"cwd": "/w", "id": "s"}}, - {"type": "response_item", "payload": {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": listing}]}}] - data = "".join(json.dumps(r) + "\n" for r in rows).encode() - self.assertEqual(codex.harvest_rollout(data)["listed"], {"deslop": "Remove AI slop.", "how": "Use for how X works."}) + {"type": "response_item", "payload": {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": developer_text}]}}] + return "".join(json.dumps(r) + "\n" for r in rows).encode() + + def test_the_lead_rollout_listing_reaches_the_harvest_without_its_file_paths(self): + data = self.harvest(self.ENTRIES + "") + self.assertEqual(codex.harvest_rollout(data)["listed"], self.LISTED) self.assertEqual(codex.harvest_rollout(data[:data.index(b"\n") + 1])["listed"], None) + def test_a_how_to_use_section_before_the_closing_tag_is_not_part_of_the_listing(self): + text = self.ENTRIES + "### How to use skills\n- Discovery: the list above is the skills available.\n" + self.assertEqual(codex.harvest_rollout(self.harvest(text))["listed"], self.LISTED) + + def test_the_listing_ends_at_the_closing_tag_when_the_next_heading_is_further_on(self): + after = ("\nFilesystem sandboxing defines which files can be read or written. " + "`sandbox_mode` is `workspace-write`: The sandbox permits reading files, and editing files in `cwd` and `writable_roots`. " + "Editing files in other directories requires approval. Network access is restricted.\n" + "- network: restricted\n" + "Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.\n" + " The writable roots are `/w/tmp`, `/w/relay`, `/w/relay/.git`, `/private/tmp`.\n" + "# Collaboration Mode: Default\n\nYou are now in Default mode.\n\n" + "## request_user_input availability\n\nUse the `request_user_input` tool only when it is listed.") + text = self.ENTRIES + "" + after + self.assertEqual(codex.harvest_rollout(self.harvest(text))["listed"], self.LISTED) + class Compaction(unittest.TestCase): def setUp(self):