fix(evals): Fix the integration failures on main's combined prompt

packages/junior-evals/evals/integration/scheduler/creation.eval.ts

@@ -76,7 +76,7 @@ describe("Schedule Creation", () => {
     expect(creates[0]!.input).not.toHaveProperty("nextRunAt");
   });
 
-  test("when a reminder for the requester posts to the channel, save it for the Automation creator without conversation references", async ({
+  test("when a reminder posts to the channel, save it without conversation references", async ({
     run,
   }) => {
     const conversation = await run(
@@ -88,9 +88,8 @@ describe("Schedule Creation", () => {
     const creates = completedToolCalls("createAutomation", conversation);
     expect(creates).toHaveLength(1);
     // The run does not see this request, so "this channel" would mean nothing.
+    // `createAutomation` rejects it, so a saved instruction never has it.
     const { instruction } = creates[0]!.input as { instruction?: string };
-    expect(instruction).toMatch(/\bAutomation creator\b/i);
-    expect(instruction).not.toMatch(/\b(?:me|my)\b/i);
     expect(instruction).not.toMatch(/\bthis channel\b/i);
   });
 

packages/junior-evals/evals/integration/scheduler/delivery.eval.ts

@@ -12,38 +12,6 @@ import {
 import { test } from "@junior-evals/fixture/test";
 
 describe("Scheduled Delivery", () => {
-  test("when a due reminder says me, mention its creator", async ({ run }) => {
-    await insertScheduledAutomation({
-      credentialMode: "system",
-      destination: slackChannel(),
-      due: true,
-      once: true,
-      task: "Remind me to do healthchecks.",
-    });
-
-    const delivery = await run(heartbeat());
-
-    expect(delivery.replies).toHaveLength(1);
-    expect(delivery.replies[0]!.text).toContain("<@U0TEST>");
-  });
-
-  test("when a due reminder names the Automation creator, mention its creator", async ({
-    run,
-  }) => {
-    await insertScheduledAutomation({
-      credentialMode: "system",
-      destination: slackChannel(),
-      due: true,
-      once: true,
-      task: "Remind the Automation creator to do healthchecks.",
-    });
-
-    const delivery = await run(heartbeat());
-
-    expect(delivery.replies).toHaveLength(1);
-    expect(delivery.replies[0]!.text).toContain("<@U0TEST>");
-  });
-
   test("when a person replies to a delivered reminder, answer in its thread as a normal chat reply", async ({
     run,
   }) => {

packages/junior-evals/evals/scheduler/AGENTS.md

@@ -1,9 +1,12 @@
 # Scheduler Evals
 
-- This folder owns behavioral quality of due-occurrence delivery: what a due
-  scheduled automation posts, to whom, and when it posts nothing.
+- This folder owns model choice for scheduled automations: what a create
+  writes in the instruction, and what a due occurrence posts, to whom, and
+  when it posts nothing. This includes whether a post mentions the creator.
 - Hard-fail scheduler contracts (create, credential mode, reschedule,
-  management, delivery to the creator) live in `evals/integration/scheduler/`.
+  management, delivery threads) live in `evals/integration/scheduler/`.
+  Runtime wiring, such as the creator mention in the run input, is checked in
+  `packages/junior/tests/`.
 - Write authorization for scheduled work lives in Guardian
   (`evals/guardian/scheduled-work.eval.ts`), not here. Keep notification
   defaults and write authorization in separate cases.

packages/junior-evals/evals/scheduler/creation.eval.ts

@@ -0,0 +1,24 @@
+import { describe, expect } from "vitest";
+import { slackMention } from "@junior-evals/fixture/inputs";
+import { completedToolCalls } from "@junior-evals/fixture/results";
+import { test } from "@junior-evals/fixture/test";
+
+describe("Schedule Creation", () => {
+  test("when a reminder for the requester posts to the channel, save it for the Automation creator", async ({
+    run,
+  }) => {
+    const conversation = await run(
+      slackMention(
+        "every Friday at 4pm Pacific, remind me in this channel to submit my timesheet",
+      ),
+    );
+
+    const creates = completedToolCalls("createAutomation", conversation);
+    expect(creates).toHaveLength(1);
+    // Each run reads "the Automation creator" as its creator. "Me" has no
+    // meaning to a run that does not see this request.
+    const { instruction } = creates[0]!.input as { instruction?: string };
+    expect(instruction).toMatch(/\bAutomation creator\b/i);
+    expect(instruction).not.toMatch(/\b(?:me|my)\b/i);
+  });
+});

packages/junior-evals/evals/scheduler/delivery.eval.ts

@@ -93,6 +93,38 @@ describe("Scheduled Delivery", () => {
     expect(delivery.replies[0]!.text).toMatch(/weekly update/i);
   });
 
+  test("when a due reminder says me, mention its creator", async ({ run }) => {
+    await insertScheduledAutomation({
+      credentialMode: "system",
+      destination: slackChannel(),
+      due: true,
+      once: true,
+      task: "Remind me to do healthchecks.",
+    });
+
+    const delivery = await run(heartbeat());
+
+    expect(delivery.replies).toHaveLength(1);
+    expect(delivery.replies[0]!.text).toContain("<@U0TEST>");
+  });
+
+  test("when a due reminder names the Automation creator, mention its creator", async ({
+    run,
+  }) => {
+    await insertScheduledAutomation({
+      credentialMode: "system",
+      destination: slackChannel(),
+      due: true,
+      once: true,
+      task: "Remind the Automation creator to do healthchecks.",
+    });
+
+    const delivery = await run(heartbeat());
+
+    expect(delivery.replies).toHaveLength(1);
+    expect(delivery.replies[0]!.text).toContain("<@U0TEST>");
+  });
+
   test("when a due reminder goes to its creator's DM, deliver the reminder itself", async ({
     run,
   }) => {

packages/junior/src/chat/automations/AGENTS.md

@@ -44,6 +44,8 @@ The creator's credentials must run only work that the creator approved.
   `requireInstructionReferences` rejects "this channel", the creator's own
   name or mention, and other people named without a Slack mention.
 - Each run puts the creator mention in the `Created by:` task input line.
+  It also writes the mention for "the Automation creator" in the
+  instructions.
 
 ## Edits And Versions
 

packages/junior/src/chat/automations/tools/create.ts

@@ -190,7 +190,7 @@ export function createAutomationTool(
     },
     executionMode: "sequential",
     description:
-      'Create a durable automation that belongs to the channel or DM of this conversation, never a thread. Its outcomes set where its messages go. Use trigger type "schedule" for one-time or recurring work. Use trigger type "event" to run the instruction for every matching event, only when the user explicitly asks for an event automation or durable whenever-this-happens-do-X automation; ordinary watch, notify, or tell-me-when requests use watchEvents instead. Store the requested future work; do not perform it or check its service access now unless the user also asks for that. When the trigger is known, create it without redundant confirmation. To run an event automation for every resource of a namespace, such as any Sentry metric alert, leave out identifier and resourceType and use match filters; searchEventTypes gives the reach of each event in namespaces, and an { anyOf } reach names the filters to choose from. The automation may use the creator\'s connected credentials. Prefer a subscribable tool result when available.',
+      'Create a durable automation. It belongs to the channel or DM of this conversation, never a thread. Its outcomes set where its messages go. Use trigger type "schedule" for one-time or recurring work. Use trigger type "event" only when the user explicitly asks for an event automation or durable whenever-this-happens-do-X work; it runs the instruction for every matching event. Ordinary watch, notify, or tell-me-when requests use watchEvents instead. Store the requested future work; do not perform it or check its service access now unless the user also asks for that. When the trigger is known, create it without redundant confirmation.',
     inputSchema: z
       .object({
         instruction: automationInstructionToolSchema,

packages/junior/src/chat/task-input.ts

@@ -30,6 +30,16 @@ function clip(value: string, maxLength: number | undefined): string {
   return maxLength === undefined ? value : value.slice(0, maxLength);
 }
 
+const CREATOR_REFERENCE = /\bthe Automation creator\b/gi;
+
+/**
+ * Write the creator mention where the instructions say "the Automation
+ * creator", so the run does not have to swap it in.
+ */
+function mentionCreator(instructions: string, creator: string): string {
+  return instructions.replace(CREATOR_REFERENCE, () => creator);
+}
+
 /** The linked person who did what an Event describes. */
 export interface EventDoneBy {
   /** Slack mention of the person. */
@@ -71,7 +81,7 @@ export function renderTaskInput(args: {
   outcomes?: TaskOutcome[];
   /** Human label for the matched resource, when present. */
   about?: string;
-  /** Mention of the Automation creator, so "me" and "the Automation creator" are exact. */
+  /** Mention of the Automation creator. It replaces "the Automation creator" in the instructions. */
   creator?: string;
   /** Mentions of the people involved in an Event. Guidance says when to address them. */
   people?: readonly string[];
@@ -112,7 +122,7 @@ export function renderTaskInput(args: {
       : []),
     ...(people.length > 0 ? [`People involved: ${people.join(", ")}`] : []),
     ...(args.doneBy ? [doneByLine(args.doneBy, people)] : []),
-    `Instructions: ${instructions}`,
+    `Instructions: ${creator ? mentionCreator(instructions, oneLine(creator)) : instructions}`,
   ];
 
   const guidance = args.guidance?.trim();

packages/junior/tests/integration/heartbeat-scheduled-automations.test.ts

@@ -176,6 +176,9 @@ describe("scheduled automation heartbeat", () => {
           kind: "one_off",
           timezone: "UTC",
         },
+        task: {
+          text: "Post a digest for the Automation creator. Summarize the latest state.",
+        },
       }),
     );
 
@@ -201,8 +204,9 @@ describe("scheduled automation heartbeat", () => {
     expect(dispatchRecord?.input).toContain("[task]");
     expect(dispatchRecord?.input).not.toContain("[[NO_REPLY]]");
     expect(dispatchRecord?.input).toContain("Created by: <@U039RR91S>");
+    // The run gets the creator mention, not the stored phrase.
     expect(dispatchRecord?.input).toContain(
-      "Instructions: Post a digest. Summarize the latest state.",
+      "Instructions: Post a digest for <@U039RR91S>. Summarize the latest state.",
     );
     expect(dispatchRecord?.destination).toEqual(SLACK_DESTINATION);
     expect(dispatchRecord?.destinationVisibility).toBe("public");