test(evals): Add behavioral evals for the review desk tools

packages/junior-evals/evals/review-desk/items.eval.ts

@@ -0,0 +1,113 @@
+import { describe, expect } from "vitest";
+import { heartbeat, slackMention } from "@junior-evals/fixture/inputs";
+import {
+  insertReviewItem,
+  insertScheduledAutomation,
+  slackChannel,
+} from "@junior-evals/fixture/insert";
+import {
+  completedToolCalls,
+  toolCallsOf,
+  toolOutput,
+} from "@junior-evals/fixture/results";
+import { test } from "@junior-evals/fixture/test";
+
+const PULL_REQUEST_URL = "https://github.com/fairsource/fair.io/pull/812";
+
+const ENTRY_REVIEW_TASK = [
+  `Pull request ${PULL_REQUEST_URL} adds the company entry for Acme to the Fair Source list.`,
+  "Its license check is unclear: Acme uses BUSL with a six-year change date.",
+  "I need to decide if we approve or reject the entry, and why.",
+  "Get that decision from me. Do not comment on or merge the pull request.",
+].join(" ");
+
+const DECISION_REQUEST = {
+  question: "Approve or reject the Acme entry?",
+  fields: [
+    {
+      id: "decision",
+      kind: "choice" as const,
+      label: "Decision",
+      options: [{ label: "Approve" }, { label: "Reject" }],
+    },
+    { id: "reason", kind: "text" as const, label: "Reason" },
+  ],
+};
+
+function itemOutput(call: Parameters<typeof toolOutput>[0]) {
+  return toolOutput(call) as {
+    created?: boolean;
+    item: { id: string; status: string };
+  };
+}
+
+describe("Review desk", () => {
+  test("when a later run finds the open item for the same pull request, update it instead of adding another", async ({
+    run,
+  }) => {
+    const destination = slackChannel();
+    const automation = await insertScheduledAutomation({
+      credentialMode: "system",
+      destination,
+      due: true,
+      task: ENTRY_REVIEW_TASK,
+    });
+    const existing = await insertReviewItem({
+      automationId: automation.id,
+      body: "Acme uses BUSL with a six-year change date.",
+      dedupeKey: "fair-source:acme-entry",
+      destination,
+      kind: "action",
+      request: DECISION_REQUEST,
+      subject: { id: PULL_REQUEST_URL, url: PULL_REQUEST_URL },
+      title: "Review the Acme entry",
+    });
+
+    const delivery = await run(heartbeat());
+
+    // The run works with the review desk. Each completed add updates the
+    // open item, so no second item starts.
+    expect(
+      toolCallsOf("addReviewItem", delivery).length +
+        toolCallsOf("searchReviewItems", delivery).length,
+    ).toBeGreaterThan(0);
+    const added = completedToolCalls("addReviewItem", delivery).map(itemOutput);
+    expect(added.filter((output) => output.created)).toEqual([]);
+    for (const output of added) {
+      expect(output.item.id).toBe(existing.id);
+    }
+  });
+
+  test("when a person says the pull request merged, close its open item with a reason", async ({
+    run,
+  }) => {
+    const destination = slackChannel();
+    const existing = await insertReviewItem({
+      body: "Acme uses BUSL with a six-year change date.",
+      dedupeKey: "fair-source:acme-entry",
+      destination,
+      kind: "action",
+      request: DECISION_REQUEST,
+      subject: { id: PULL_REQUEST_URL, url: PULL_REQUEST_URL },
+      title: "Review the Acme entry",
+    });
+
+    const reply = await run(
+      slackMention(
+        `${PULL_REQUEST_URL} got merged, so nobody needs to review the Acme entry anymore. Clean it up on the review desk.`,
+        { channel: destination },
+      ),
+    );
+
+    const closes = completedToolCalls("updateReviewItem", reply).filter(
+      (call) => (call.input as { status?: string }).status,
+    );
+    expect(closes).toHaveLength(1);
+    expect(closes[0]!.input).toMatchObject({
+      id: existing.id,
+      reason: expect.any(String),
+      status: expect.stringMatching(/^(done|dismissed)$/),
+    });
+    expect(itemOutput(closes[0]!).item.status).toMatch(/^(done|dismissed)$/);
+  });
+});

packages/junior-evals/src/fixture/insert.ts

@@ -18,6 +18,10 @@ import { createWatch } from "@/chat/events/store";
 import type { EventAutomation } from "@/chat/event-automations/types";
 import { upsertIdentity } from "@/chat/identities/sql";
 import { createPluginEmbedder } from "@/chat/plugins/model";
+import {
+  insertReviewItem as insertStoredReviewItem,
+  type ReviewItemContent,
+} from "@/chat/review-items/store";
 import { saveScheduledAutomation } from "@/chat/scheduled-automations/tasks";
 import {
   SCHEDULED_AUTOMATION_SYSTEM_ACTOR,
@@ -108,9 +112,12 @@ async function insertSlackIdentity(author: Required<SlackAuthor>) {
  * runs do not record channel visibility, so without it their Conversations in
  * the channel are private.
  */
-async function insertCreationThread(destination: SlackChannel) {
+async function insertCreationThread(
+  destination: SlackChannel,
+): Promise<string> {
+  const conversationId = `slack:${destination.channelId}:${FIXTURE_MESSAGE_TS}`;
   await getConversationStore().recordActivity({
-    conversationId: `slack:${destination.channelId}:${FIXTURE_MESSAGE_TS}`,
+    conversationId,
     destination,
     sessionSource: createSlackSource({
       channelId: destination.channelId,
@@ -121,6 +128,7 @@ async function insertCreationThread(destination: SlackChannel) {
     source: "slack",
     visibility: "public",
   });
+  return conversationId;
 }
 
 /**
@@ -225,6 +233,47 @@ export async function insertEventAutomation(args: {
   return { id };
 }
 
+/**
+ * Store an open Review item that an earlier run added in a public thread of
+ * `destination`. It is assigned to the Slack person `assignee`. With
+ * `automationId`, later runs of that Automation can see and change it.
+ */
+export async function insertReviewItem(args: {
+  assignee?: SlackAuthor;
+  automationId?: string;
+  body: string;
+  dedupeKey: string;
+  destination: SlackChannel;
+  kind: ReviewItemContent["kind"];
+  request?: ReviewItemContent["request"];
+  subject?: ReviewItemContent["subject"];
+  title: string;
+}): Promise<{ id: string }> {
+  const identity = await insertSlackIdentity(resolveAuthor(args.assignee));
+  if (!identity.userId) {
+    throw new Error("insertReviewItem() needs an assignee with a Junior user");
+  }
+  const conversationId = await insertCreationThread(args.destination);
+  const item = await insertStoredReviewItem(getDb(), {
+    addedByConversationId: conversationId,
+    assignedToEveryone: false,
+    assigneeUserIds: [identity.userId],
+    body: args.body,
+    conversationIds: [conversationId],
+    dedupeKey: args.dedupeKey,
+    kind: args.kind,
+    nowMs: Date.now() - 60_000,
+    tenant: { provider: "slack", providerTenantId: args.destination.teamId },
+    title: args.title,
+    visibility: "public",
+    ...(args.automationId ? { automationId: args.automationId } : undefined),
+    ...(args.request ? { request: args.request } : undefined),
+    ...(args.subject ? { subject: args.subject } : undefined),
+  });
+  if (!item) throw new Error(`Review item ${args.dedupeKey} is already open`);
+  return { id: item.id };
+}
+
 /**
  * Store a watch on a Conversation of the test. It lasts two weeks. A later
  * matching event, such as `conversation.continue(githubWebhook(...))`,