diff --git a/packages/junior-evals/evals/integration/scheduler/creation.eval.ts b/packages/junior-evals/evals/integration/scheduler/creation.eval.ts index 50b4e98c60..aac9a6314d 100644 --- a/packages/junior-evals/evals/integration/scheduler/creation.eval.ts +++ b/packages/junior-evals/evals/integration/scheduler/creation.eval.ts @@ -76,7 +76,7 @@ describe("Schedule Creation", () => { expect(creates[0]!.input).not.toHaveProperty("nextRunAt"); }); - test("when a reminder for the requester posts to the channel, save it for the Automation creator without conversation references", async ({ + test("when a reminder posts to the channel, save it without conversation references", async ({ run, }) => { const conversation = await run( @@ -88,9 +88,8 @@ describe("Schedule Creation", () => { const creates = completedToolCalls("createAutomation", conversation); expect(creates).toHaveLength(1); // The run does not see this request, so "this channel" would mean nothing. + // `createAutomation` rejects it, so a saved instruction never has it. const { instruction } = creates[0]!.input as { instruction?: string }; - expect(instruction).toMatch(/\bAutomation creator\b/i); - expect(instruction).not.toMatch(/\b(?:me|my)\b/i); expect(instruction).not.toMatch(/\bthis channel\b/i); }); diff --git a/packages/junior-evals/evals/integration/scheduler/delivery.eval.ts b/packages/junior-evals/evals/integration/scheduler/delivery.eval.ts index 7d268ff61d..a5494e3ee2 100644 --- a/packages/junior-evals/evals/integration/scheduler/delivery.eval.ts +++ b/packages/junior-evals/evals/integration/scheduler/delivery.eval.ts @@ -12,38 +12,6 @@ import { import { test } from "@junior-evals/fixture/test"; describe("Scheduled Delivery", () => { - test("when a due reminder says me, mention its creator", async ({ run }) => { - await insertScheduledAutomation({ - credentialMode: "system", - destination: slackChannel(), - due: true, - once: true, - task: "Remind me to do healthchecks.", - }); - - const delivery = await run(heartbeat()); - - expect(delivery.replies).toHaveLength(1); - expect(delivery.replies[0]!.text).toContain("<@U0TEST>"); - }); - - test("when a due reminder names the Automation creator, mention its creator", async ({ - run, - }) => { - await insertScheduledAutomation({ - credentialMode: "system", - destination: slackChannel(), - due: true, - once: true, - task: "Remind the Automation creator to do healthchecks.", - }); - - const delivery = await run(heartbeat()); - - expect(delivery.replies).toHaveLength(1); - expect(delivery.replies[0]!.text).toContain("<@U0TEST>"); - }); - test("when a person replies to a delivered reminder, answer in its thread as a normal chat reply", async ({ run, }) => { diff --git a/packages/junior-evals/evals/scheduler/AGENTS.md b/packages/junior-evals/evals/scheduler/AGENTS.md index 3dae9cb1d5..0db31f9188 100644 --- a/packages/junior-evals/evals/scheduler/AGENTS.md +++ b/packages/junior-evals/evals/scheduler/AGENTS.md @@ -1,9 +1,12 @@ # Scheduler Evals -- This folder owns behavioral quality of due-occurrence delivery: what a due - scheduled automation posts, to whom, and when it posts nothing. +- This folder owns model choice for scheduled automations: what a create + writes in the instruction, and what a due occurrence posts, to whom, and + when it posts nothing. This includes whether a post mentions the creator. - Hard-fail scheduler contracts (create, credential mode, reschedule, - management, delivery to the creator) live in `evals/integration/scheduler/`. + management, delivery threads) live in `evals/integration/scheduler/`. + Runtime wiring, such as the creator mention in the run input, is checked in + `packages/junior/tests/`. - Write authorization for scheduled work lives in Guardian (`evals/guardian/scheduled-work.eval.ts`), not here. Keep notification defaults and write authorization in separate cases. diff --git a/packages/junior-evals/evals/scheduler/creation.eval.ts b/packages/junior-evals/evals/scheduler/creation.eval.ts new file mode 100644 index 0000000000..8cd4c7587b --- /dev/null +++ b/packages/junior-evals/evals/scheduler/creation.eval.ts @@ -0,0 +1,24 @@ +import { describe, expect } from "vitest"; +import { slackMention } from "@junior-evals/fixture/inputs"; +import { completedToolCalls } from "@junior-evals/fixture/results"; +import { test } from "@junior-evals/fixture/test"; + +describe("Schedule Creation", () => { + test("when a reminder for the requester posts to the channel, save it for the Automation creator", async ({ + run, + }) => { + const conversation = await run( + slackMention( + "every Friday at 4pm Pacific, remind me in this channel to submit my timesheet", + ), + ); + + const creates = completedToolCalls("createAutomation", conversation); + expect(creates).toHaveLength(1); + // Each run reads "the Automation creator" as its creator. "Me" has no + // meaning to a run that does not see this request. + const { instruction } = creates[0]!.input as { instruction?: string }; + expect(instruction).toMatch(/\bAutomation creator\b/i); + expect(instruction).not.toMatch(/\b(?:me|my)\b/i); + }); +}); diff --git a/packages/junior-evals/evals/scheduler/delivery.eval.ts b/packages/junior-evals/evals/scheduler/delivery.eval.ts index c63329bb7f..ec66a35ab8 100644 --- a/packages/junior-evals/evals/scheduler/delivery.eval.ts +++ b/packages/junior-evals/evals/scheduler/delivery.eval.ts @@ -93,6 +93,38 @@ describe("Scheduled Delivery", () => { expect(delivery.replies[0]!.text).toMatch(/weekly update/i); }); + test("when a due reminder says me, mention its creator", async ({ run }) => { + await insertScheduledAutomation({ + credentialMode: "system", + destination: slackChannel(), + due: true, + once: true, + task: "Remind me to do healthchecks.", + }); + + const delivery = await run(heartbeat()); + + expect(delivery.replies).toHaveLength(1); + expect(delivery.replies[0]!.text).toContain("<@U0TEST>"); + }); + + test("when a due reminder names the Automation creator, mention its creator", async ({ + run, + }) => { + await insertScheduledAutomation({ + credentialMode: "system", + destination: slackChannel(), + due: true, + once: true, + task: "Remind the Automation creator to do healthchecks.", + }); + + const delivery = await run(heartbeat()); + + expect(delivery.replies).toHaveLength(1); + expect(delivery.replies[0]!.text).toContain("<@U0TEST>"); + }); + test("when a due reminder goes to its creator's DM, deliver the reminder itself", async ({ run, }) => { diff --git a/packages/junior/src/chat/automations/AGENTS.md b/packages/junior/src/chat/automations/AGENTS.md index dbbb79ea3d..f7f51958c3 100644 --- a/packages/junior/src/chat/automations/AGENTS.md +++ b/packages/junior/src/chat/automations/AGENTS.md @@ -44,6 +44,8 @@ The creator's credentials must run only work that the creator approved. `requireInstructionReferences` rejects "this channel", the creator's own name or mention, and other people named without a Slack mention. - Each run puts the creator mention in the `Created by:` task input line. + It also writes the mention for "the Automation creator" in the + instructions. ## Edits And Versions diff --git a/packages/junior/src/chat/automations/tools/create.ts b/packages/junior/src/chat/automations/tools/create.ts index eb337659d5..c210fef679 100644 --- a/packages/junior/src/chat/automations/tools/create.ts +++ b/packages/junior/src/chat/automations/tools/create.ts @@ -190,7 +190,7 @@ export function createAutomationTool( }, executionMode: "sequential", description: - 'Create a durable automation that belongs to the channel or DM of this conversation, never a thread. Its outcomes set where its messages go. Use trigger type "schedule" for one-time or recurring work. Use trigger type "event" to run the instruction for every matching event, only when the user explicitly asks for an event automation or durable whenever-this-happens-do-X automation; ordinary watch, notify, or tell-me-when requests use watchEvents instead. Store the requested future work; do not perform it or check its service access now unless the user also asks for that. When the trigger is known, create it without redundant confirmation. To run an event automation for every resource of a namespace, such as any Sentry metric alert, leave out identifier and resourceType and use match filters; searchEventTypes gives the reach of each event in namespaces, and an { anyOf } reach names the filters to choose from. The automation may use the creator\'s connected credentials. Prefer a subscribable tool result when available.', + 'Create a durable automation. It belongs to the channel or DM of this conversation, never a thread. Its outcomes set where its messages go. Use trigger type "schedule" for one-time or recurring work. Use trigger type "event" only when the user explicitly asks for an event automation or durable whenever-this-happens-do-X work; it runs the instruction for every matching event. Ordinary watch, notify, or tell-me-when requests use watchEvents instead. Store the requested future work; do not perform it or check its service access now unless the user also asks for that. When the trigger is known, create it without redundant confirmation.', inputSchema: z .object({ instruction: automationInstructionToolSchema, diff --git a/packages/junior/src/chat/task-input.ts b/packages/junior/src/chat/task-input.ts index ec0f730b32..a2029f836c 100644 --- a/packages/junior/src/chat/task-input.ts +++ b/packages/junior/src/chat/task-input.ts @@ -30,6 +30,16 @@ function clip(value: string, maxLength: number | undefined): string { return maxLength === undefined ? value : value.slice(0, maxLength); } +const CREATOR_REFERENCE = /\bthe Automation creator\b/gi; + +/** + * Write the creator mention where the instructions say "the Automation + * creator", so the run does not have to swap it in. + */ +function mentionCreator(instructions: string, creator: string): string { + return instructions.replace(CREATOR_REFERENCE, () => creator); +} + /** The linked person who did what an Event describes. */ export interface EventDoneBy { /** Slack mention of the person. */ @@ -71,7 +81,7 @@ export function renderTaskInput(args: { outcomes?: TaskOutcome[]; /** Human label for the matched resource, when present. */ about?: string; - /** Mention of the Automation creator, so "me" and "the Automation creator" are exact. */ + /** Mention of the Automation creator. It replaces "the Automation creator" in the instructions. */ creator?: string; /** Mentions of the people involved in an Event. Guidance says when to address them. */ people?: readonly string[]; @@ -112,7 +122,7 @@ export function renderTaskInput(args: { : []), ...(people.length > 0 ? [`People involved: ${people.join(", ")}`] : []), ...(args.doneBy ? [doneByLine(args.doneBy, people)] : []), - `Instructions: ${instructions}`, + `Instructions: ${creator ? mentionCreator(instructions, oneLine(creator)) : instructions}`, ]; const guidance = args.guidance?.trim(); diff --git a/packages/junior/tests/integration/heartbeat-scheduled-automations.test.ts b/packages/junior/tests/integration/heartbeat-scheduled-automations.test.ts index b0cd86fd2a..b219beb718 100644 --- a/packages/junior/tests/integration/heartbeat-scheduled-automations.test.ts +++ b/packages/junior/tests/integration/heartbeat-scheduled-automations.test.ts @@ -176,6 +176,9 @@ describe("scheduled automation heartbeat", () => { kind: "one_off", timezone: "UTC", }, + task: { + text: "Post a digest for the Automation creator. Summarize the latest state.", + }, }), ); @@ -201,8 +204,9 @@ describe("scheduled automation heartbeat", () => { expect(dispatchRecord?.input).toContain("[task]"); expect(dispatchRecord?.input).not.toContain("[[NO_REPLY]]"); expect(dispatchRecord?.input).toContain("Created by: <@U039RR91S>"); + // The run gets the creator mention, not the stored phrase. expect(dispatchRecord?.input).toContain( - "Instructions: Post a digest. Summarize the latest state.", + "Instructions: Post a digest for <@U039RR91S>. Summarize the latest state.", ); expect(dispatchRecord?.destination).toEqual(SLACK_DESTINATION); expect(dispatchRecord?.destinationVisibility).toBe("public");