diff --git a/packages/junior-evals/evals/review-desk/items.eval.ts b/packages/junior-evals/evals/review-desk/items.eval.ts new file mode 100644 index 0000000000..e535939a3c --- /dev/null +++ b/packages/junior-evals/evals/review-desk/items.eval.ts @@ -0,0 +1,113 @@ +import { describe, expect } from "vitest"; +import { heartbeat, slackMention } from "@junior-evals/fixture/inputs"; +import { + insertReviewItem, + insertScheduledAutomation, + slackChannel, +} from "@junior-evals/fixture/insert"; +import { + completedToolCalls, + toolCallsOf, + toolOutput, +} from "@junior-evals/fixture/results"; +import { test } from "@junior-evals/fixture/test"; + +const PULL_REQUEST_URL = "https://github.com/fairsource/fair.io/pull/812"; + +const ENTRY_REVIEW_TASK = [ + `Pull request ${PULL_REQUEST_URL} adds the company entry for Acme to the Fair Source list.`, + "Its license check is unclear: Acme uses BUSL with a six-year change date.", + "I need to decide if we approve or reject the entry, and why.", + "Get that decision from me. Do not comment on or merge the pull request.", +].join(" "); + +const DECISION_REQUEST = { + question: "Approve or reject the Acme entry?", + fields: [ + { + id: "decision", + kind: "choice" as const, + label: "Decision", + options: [{ label: "Approve" }, { label: "Reject" }], + }, + { id: "reason", kind: "text" as const, label: "Reason" }, + ], +}; + +function itemOutput(call: Parameters[0]) { + return toolOutput(call) as { + created?: boolean; + item: { id: string; status: string }; + }; +} + +describe("Review desk", () => { + test("when a later run finds the open item for the same pull request, update it instead of adding another", async ({ + run, + }) => { + const destination = slackChannel(); + const automation = await insertScheduledAutomation({ + credentialMode: "system", + destination, + due: true, + task: ENTRY_REVIEW_TASK, + }); + const existing = await insertReviewItem({ + automationId: automation.id, + body: "Acme uses BUSL with a six-year change date.", + dedupeKey: "fair-source:acme-entry", + destination, + kind: "action", + request: DECISION_REQUEST, + subject: { id: PULL_REQUEST_URL, url: PULL_REQUEST_URL }, + title: "Review the Acme entry", + }); + + const delivery = await run(heartbeat()); + + // The run works with the review desk. Each completed add updates the + // open item, so no second item starts. + expect( + toolCallsOf("addReviewItem", delivery).length + + toolCallsOf("searchReviewItems", delivery).length, + ).toBeGreaterThan(0); + const added = completedToolCalls("addReviewItem", delivery).map(itemOutput); + expect(added.filter((output) => output.created)).toEqual([]); + for (const output of added) { + expect(output.item.id).toBe(existing.id); + } + }); + + test("when a person says the pull request merged, close its open item with a reason", async ({ + run, + }) => { + const destination = slackChannel(); + const existing = await insertReviewItem({ + body: "Acme uses BUSL with a six-year change date.", + dedupeKey: "fair-source:acme-entry", + destination, + kind: "action", + request: DECISION_REQUEST, + subject: { id: PULL_REQUEST_URL, url: PULL_REQUEST_URL }, + title: "Review the Acme entry", + }); + + const reply = await run( + slackMention( + `${PULL_REQUEST_URL} got merged, so nobody needs to review the Acme entry anymore. Clean it up on the review desk.`, + { channel: destination }, + ), + ); + + const closes = completedToolCalls("updateReviewItem", reply).filter( + (call) => (call.input as { status?: string }).status, + ); + expect(closes).toHaveLength(1); + expect(closes[0]!.input).toMatchObject({ + id: existing.id, + reason: expect.any(String), + status: expect.stringMatching(/^(done|dismissed)$/), + }); + expect(itemOutput(closes[0]!).item.status).toMatch(/^(done|dismissed)$/); + }); +}); diff --git a/packages/junior-evals/src/fixture/insert.ts b/packages/junior-evals/src/fixture/insert.ts index 52f24c76f8..298c1a8ac1 100644 --- a/packages/junior-evals/src/fixture/insert.ts +++ b/packages/junior-evals/src/fixture/insert.ts @@ -18,6 +18,10 @@ import { createWatch } from "@/chat/events/store"; import type { EventAutomation } from "@/chat/event-automations/types"; import { upsertIdentity } from "@/chat/identities/sql"; import { createPluginEmbedder } from "@/chat/plugins/model"; +import { + insertReviewItem as insertStoredReviewItem, + type ReviewItemContent, +} from "@/chat/review-items/store"; import { saveScheduledAutomation } from "@/chat/scheduled-automations/tasks"; import { SCHEDULED_AUTOMATION_SYSTEM_ACTOR, @@ -108,9 +112,12 @@ async function insertSlackIdentity(author: Required) { * runs do not record channel visibility, so without it their Conversations in * the channel are private. */ -async function insertCreationThread(destination: SlackChannel) { +async function insertCreationThread( + destination: SlackChannel, +): Promise { + const conversationId = `slack:${destination.channelId}:${FIXTURE_MESSAGE_TS}`; await getConversationStore().recordActivity({ - conversationId: `slack:${destination.channelId}:${FIXTURE_MESSAGE_TS}`, + conversationId, destination, sessionSource: createSlackSource({ channelId: destination.channelId, @@ -121,6 +128,7 @@ async function insertCreationThread(destination: SlackChannel) { source: "slack", visibility: "public", }); + return conversationId; } /** @@ -225,6 +233,47 @@ export async function insertEventAutomation(args: { return { id }; } +/** + * Store an open Review item that an earlier run added in a public thread of + * `destination`. It is assigned to the Slack person `assignee`. With + * `automationId`, later runs of that Automation can see and change it. + */ +export async function insertReviewItem(args: { + assignee?: SlackAuthor; + automationId?: string; + body: string; + dedupeKey: string; + destination: SlackChannel; + kind: ReviewItemContent["kind"]; + request?: ReviewItemContent["request"]; + subject?: ReviewItemContent["subject"]; + title: string; +}): Promise<{ id: string }> { + const identity = await insertSlackIdentity(resolveAuthor(args.assignee)); + if (!identity.userId) { + throw new Error("insertReviewItem() needs an assignee with a Junior user"); + } + const conversationId = await insertCreationThread(args.destination); + const item = await insertStoredReviewItem(getDb(), { + addedByConversationId: conversationId, + assignedToEveryone: false, + assigneeUserIds: [identity.userId], + body: args.body, + conversationIds: [conversationId], + dedupeKey: args.dedupeKey, + kind: args.kind, + nowMs: Date.now() - 60_000, + tenant: { provider: "slack", providerTenantId: args.destination.teamId }, + title: args.title, + visibility: "public", + ...(args.automationId ? { automationId: args.automationId } : undefined), + ...(args.request ? { request: args.request } : undefined), + ...(args.subject ? { subject: args.subject } : undefined), + }); + if (!item) throw new Error(`Review item ${args.dedupeKey} is already open`); + return { id: item.id }; +} + /** * Store a watch on a Conversation of the test. It lasts two weeks. A later * matching event, such as `conversation.continue(githubWebhook(...))`,