diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..491e4bf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + - run: npm ci + - run: npx playwright install --with-deps chromium + - run: npm run check diff --git a/README.md b/README.md index ed15cd6..791f6a2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ LLM-driven computer-use system that discovers UI workflows, records them as reus ## Status -The capability schema and replay result contract are in place. Discovery, replay, the target app, and human handoff are not built yet. +The capability schema, local core-servicing app, deterministic replay, policy, and human handoff are in place. Discovery is not built yet. ## Setup @@ -12,11 +12,17 @@ Requires Node 22.12+. ```bash npm install -npm test -npm run typecheck +npx playwright install chromium +npm run check ``` -No API keys needed for the current code. +Start the local target app (no API keys): + +```bash +npm run app +``` + +It listens on `http://127.0.0.1:4173/`. Member `10001` has a savings balance; any other id returns "Member not found". ## Demo diff --git a/apps/core-servicing/server.ts b/apps/core-servicing/server.ts new file mode 100644 index 0000000..659b18e --- /dev/null +++ b/apps/core-servicing/server.ts @@ -0,0 +1,144 @@ +import http from "node:http"; +import { fileURLToPath } from "node:url"; + +export const DEFAULT_PORT = 4173; + +type Member = { + displayName: string; + accounts: { type: string; balance: string }[]; +}; + +/** Synthetic members only — no real PII. */ +export const MEMBERS: Record = { + "10001": { + displayName: "Alex Rivera", + accounts: [ + { type: "Checking", balance: "$88.12" }, + { type: "Savings", balance: "$1,240.50" }, + ], + }, + "10002": { + displayName: "Jordan Lee", + accounts: [{ type: "Savings", balance: "$50.00" }], + }, +}; + +export function createServer(): http.Server { + return http.createServer((req, res) => { + const url = new URL(req.url ?? "/", "http://127.0.0.1"); + if (req.method !== "GET") { + res.writeHead(405, { Allow: "GET" }); + res.end("Method not allowed"); + return; + } + if (url.pathname === "/" || url.pathname === "/index.html") { + send(res, searchPage()); + return; + } + if (url.pathname === "/member") { + send(res, memberPage(url.searchParams.get("id") ?? "")); + return; + } + res.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" }); + res.end("Not found"); + }); +} + +function send(res: http.ServerResponse, html: string) { + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + res.end(html); +} + +function escapeHtml(value: string) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +function shell(body: string) { + return ` + + + +Core Servicing + + + +
+
+
+${body} +
+
+ +`; +} + +function searchPage(message = "") { + const notice = message + ? `${escapeHtml(message)}` + : ""; + return shell(` +
+ +${notice} + + + + + + + + +
+
`); +} + +function memberPage(rawId: string) { + const id = rawId.trim(); + if (!id) { + return searchPage("Member ID is required"); + } + const member = MEMBERS[id]; + if (!member) { + return searchPage("Member not found"); + } + const rows = member.accounts + .map( + (account) => + `${escapeHtml(account.type)}${escapeHtml(account.balance)}`, + ) + .join(""); + return shell(` +

Member detail

+ + + +
Member${escapeHtml(id)}
Name${escapeHtml(member.displayName)}
+
+ + + ${rows} +
AccountBalance
+

New search

`); +} + +const isDirectRun = + process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (isDirectRun) { + const port = Number(process.env.PORT ?? DEFAULT_PORT); + createServer().listen(port, "127.0.0.1", () => { + process.stdout.write( + `Core servicing listening on http://127.0.0.1:${port}/\n`, + ); + }); +} diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..f28076d --- /dev/null +++ b/biome.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.5.8/schema.json", + "files": { + "includes": [ + "**", + "!node_modules", + "!dist", + "!coverage", + "!package-lock.json" + ] + }, + "formatter": { + "indentStyle": "space", + "indentWidth": 2 + } +} diff --git a/package-lock.json b/package-lock.json index 98f4042..1f3708d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,9 +8,11 @@ "name": "computer-use-capability-engine", "version": "0.1.0", "dependencies": { + "playwright": "1.62.1", "zod": "4.4.3" }, "devDependencies": { + "@biomejs/biome": "2.5.8", "@types/node": "26.2.0", "typescript": "7.0.2", "vitest": "4.1.10" @@ -19,6 +21,169 @@ "node": ">=22.12.0" } }, + "node_modules/@biomejs/biome": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.8.tgz", + "integrity": "sha512-aeAeeJB9fSDc7Gq+2GqpQxA0qBj6gj1k2R6L1cYqGePKP/baIq1WX8y6B+D+nRsO5ViQL22K/8IwbqERW0q1nw==", + "dev": true, + "license": "MIT OR Apache-2.0", + "bin": { + "biome": "bin/biome" + }, + "engines": { + "node": ">=14.21.3" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/biome" + }, + "optionalDependencies": { + "@biomejs/cli-darwin-arm64": "2.5.8", + "@biomejs/cli-darwin-x64": "2.5.8", + "@biomejs/cli-linux-arm64": "2.5.8", + "@biomejs/cli-linux-arm64-musl": "2.5.8", + "@biomejs/cli-linux-x64": "2.5.8", + "@biomejs/cli-linux-x64-musl": "2.5.8", + "@biomejs/cli-win32-arm64": "2.5.8", + "@biomejs/cli-win32-x64": "2.5.8" + } + }, + "node_modules/@biomejs/cli-darwin-arm64": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.8.tgz", + "integrity": "sha512-mk1QON9PHllvvLN5gU3f4rMxeh4syK5p9OvKyWH6/W8ueh04uaC8TUXXByhGufWf/y5mQc03ZLM45zU+cmqMjA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-darwin-x64": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.8.tgz", + "integrity": "sha512-bsGwFMBNyHPyiLSsQcZJxdoRrg1V4JL+d7wEsvUBczlP9U9lwM+7mzQHxI4o1mhBsTmdOBbAb6fHU3Z3snN45w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.8.tgz", + "integrity": "sha512-XmFiA0WPYFC+uiUDC8WRFzAIH9bo7vwQLav38Uoq4ETC+T/+uBi0TsYGJECkugY3r8USl3jc+Ae2/irAF6F2lQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64-musl": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.8.tgz", + "integrity": "sha512-VcJNbstduTHx83NGAdhp78/JOcP45BZHXL7yNsfI1uGzdUgegAz2s+mSoT7wK6PBNzLoqG0zDOXaz/RQYVtSiw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.8.tgz", + "integrity": "sha512-S5wcm9OBDvLHodD4PUaN488hCpco9QD/9ZxuYJiw4euWtr/oQvLR72z2ixItH8Wd5BCm6FZaeb+YNvOoM1xHtQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64-musl": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.8.tgz", + "integrity": "sha512-kKmiyokeISRGq2FLwvr+TzsgBusfxaZ0FZNLcOYOpCK/78tRrEjeEBLvq3xLZMpqbANgJdRPI7vZX8ZL37u9/w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-arm64": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.8.tgz", + "integrity": "sha512-nILH0mzm3Hi3iEdd7o7GpB8kBR/mSQwfQG/tyBqyNrY2GFtcgwfV9nV8xLmbtUpMNY/Oi0Ml1XgfR4flOdq+AA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-x64": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.8.tgz", + "integrity": "sha512-I2czzXTY61f3nFJxXoMDq80t7MivxDEnCjE+8sDKoFfcKMaoQdkqhIFQ3KyY0XLzeSpUBYeNAXgD+iOV/BU0VA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -1204,6 +1369,50 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.26", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", diff --git a/package.json b/package.json index f176418..f36075b 100644 --- a/package.json +++ b/package.json @@ -7,14 +7,19 @@ "node": ">=22.12.0" }, "scripts": { + "app": "node apps/core-servicing/server.ts", + "lint": "biome check .", + "typecheck": "tsc --noEmit", "test": "vitest run", "test:watch": "vitest", - "typecheck": "tsc --noEmit" + "check": "npm run typecheck && npm run lint && npm test" }, "dependencies": { + "playwright": "1.62.1", "zod": "4.4.3" }, "devDependencies": { + "@biomejs/biome": "2.5.8", "@types/node": "26.2.0", "typescript": "7.0.2", "vitest": "4.1.10" diff --git a/src/handoff/prompt.ts b/src/handoff/prompt.ts new file mode 100644 index 0000000..d7bdc42 --- /dev/null +++ b/src/handoff/prompt.ts @@ -0,0 +1,57 @@ +import { createInterface } from "node:readline/promises"; +import type { RunSession } from "../session/session"; +import type { Handoff, HandoffOutcome, InterventionRequest } from "./types"; + +/** + * Bare operator surface: the headed browser is the console. + * Prints the intervention and waits for skip | retry | abort on stdin. + */ +export class PromptOperator implements Handoff { + async intervene( + session: RunSession, + request: InterventionRequest, + ): Promise { + if (session.owner !== "human") { + throw new Error( + "operator called while automation still owns the session", + ); + } + process.stdout.write( + [ + "HITL: automation paused; use the live browser window.", + `capability=${request.capabilityId} step=${request.step.id}`, + `reason=${request.reason}`, + request.screenshotPath ? `screenshot=${request.screenshotPath}` : "", + "Type skip (human finished the step), retry, or abort, then Enter.", + "", + ] + .filter(Boolean) + .join("\n"), + ); + const rl = createInterface({ + input: process.stdin, + output: process.stdout, + }); + try { + const line = (await rl.question("> ")).trim().toLowerCase(); + if (line === "retry") { + return { + actions: [{ type: "other", detail: "stdin:retry" }], + resume: "retry_step", + }; + } + if (line === "abort") { + return { + actions: [{ type: "other", detail: "stdin:abort" }], + resume: "abort", + }; + } + return { + actions: [{ type: "other", detail: `stdin:${line || "skip"}` }], + resume: "skip_step", + }; + } finally { + rl.close(); + } + } +} diff --git a/src/handoff/scripted.ts b/src/handoff/scripted.ts new file mode 100644 index 0000000..8b3a6a5 --- /dev/null +++ b/src/handoff/scripted.ts @@ -0,0 +1,21 @@ +import type { RunSession } from "../session/session"; +import type { Handoff, HandoffOutcome, InterventionRequest } from "./types"; + +/** Test/mock operator: acts on the same session surface, then signals resume. */ +export class ScriptedOperator implements Handoff { + constructor( + private readonly run: ( + session: RunSession, + request: InterventionRequest, + ) => Promise | HandoffOutcome, + ) {} + + async intervene(session: RunSession, request: InterventionRequest) { + if (session.owner !== "human") { + throw new Error( + "operator called while automation still owns the session", + ); + } + return this.run(session, request); + } +} diff --git a/src/handoff/types.ts b/src/handoff/types.ts new file mode 100644 index 0000000..413eae0 --- /dev/null +++ b/src/handoff/types.ts @@ -0,0 +1,30 @@ +import type { Step } from "../schema/capability"; +import type { RunSession } from "../session/session"; +import type { Observation } from "../surface/surface"; + +export type ResumeDecision = "skip_step" | "retry_step" | "abort"; + +export type HumanAction = { + type: "click" | "fill" | "select" | "dismiss" | "other"; + detail: string; +}; + +export type InterventionRequest = { + capabilityId: string; + step: Step; + reason: string; + observation: Observation; + screenshotPath?: string; +}; + +export type HandoffOutcome = { + actions: HumanAction[]; + resume: ResumeDecision; +}; + +export type Handoff = { + intervene( + session: RunSession, + request: InterventionRequest, + ): Promise; +}; diff --git a/src/policy/authorize.ts b/src/policy/authorize.ts new file mode 100644 index 0000000..0f4e706 --- /dev/null +++ b/src/policy/authorize.ts @@ -0,0 +1,67 @@ +import type { Step } from "../schema/capability"; +import type { Policy } from "./schema"; + +export type Denial = { + kind: "block" | "escalate"; + reason: string; +}; + +export function authorize( + step: Step, + url: string | undefined, + policy: Policy, +): Denial | null { + if (!policy.allowedActions.includes(step.action)) { + return { + kind: "block", + reason: `action "${step.action}" is not allowlisted`, + }; + } + + if (step.action === "navigate" && url) { + const denied = authorizeUrl(url, policy); + if (denied) return denied; + } + + if (step.risk === "irreversible") { + return { + kind: policy.onIrreversible, + reason: `irreversible step "${step.id}"`, + }; + } + + return null; +} + +export function authorizeUrl(url: string, policy: Policy): Denial | null { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return { kind: "block", reason: `invalid url "${url}"` }; + } + + if (!policy.allowedHosts.includes(parsed.hostname)) { + return { + kind: "block", + reason: `host "${parsed.hostname}" is not allowlisted`, + }; + } + + const allowedPath = policy.allowedPathPrefixes.some((prefix) => + pathAllowed(parsed.pathname, prefix), + ); + if (!allowedPath) { + return { + kind: "block", + reason: `path "${parsed.pathname}" is not allowlisted`, + }; + } + + return null; +} + +function pathAllowed(pathname: string, prefix: string): boolean { + if (prefix === "/") return true; + return pathname === prefix || pathname.startsWith(`${prefix}/`); +} diff --git a/src/policy/redact.ts b/src/policy/redact.ts new file mode 100644 index 0000000..852fd9e --- /dev/null +++ b/src/policy/redact.ts @@ -0,0 +1,18 @@ +import type { Parameter } from "../schema/capability"; + +const SENSITIVE = new Set(["identifier", "financial", "secret", "full_pii"]); + +export function redactText( + text: string, + params: Record, + fields: Parameter[], +): string { + let redacted = text; + for (const field of fields) { + if (!SENSITIVE.has(field.sensitivity)) continue; + const value = params[field.name]; + if (!value) continue; + redacted = redacted.split(value).join(`[${field.name}]`); + } + return redacted; +} diff --git a/src/policy/schema.ts b/src/policy/schema.ts new file mode 100644 index 0000000..43adf61 --- /dev/null +++ b/src/policy/schema.ts @@ -0,0 +1,43 @@ +import { z } from "zod"; + +export const PolicySchema = z.strictObject({ + allowedHosts: z.array(z.string().min(1)).min(1), + allowedPathPrefixes: z.array(z.string().min(1)).default(["/"]), + allowedActions: z + .array( + z.enum([ + "navigate", + "click", + "fill", + "select", + "extract", + "assert", + "dismiss", + "wait", + ]), + ) + .min(1), + onIrreversible: z.enum(["escalate", "block"]), +}); + +export type Policy = z.infer; + +export function parsePolicy(data: unknown): Policy { + return PolicySchema.parse(data); +} + +export const DEFAULT_POLICY: Policy = { + allowedHosts: ["127.0.0.1", "localhost"], + allowedPathPrefixes: ["/"], + allowedActions: [ + "navigate", + "click", + "fill", + "select", + "extract", + "assert", + "dismiss", + "wait", + ], + onIrreversible: "escalate", +}; diff --git a/src/replay/engine.ts b/src/replay/engine.ts new file mode 100644 index 0000000..bb1c2c2 --- /dev/null +++ b/src/replay/engine.ts @@ -0,0 +1,388 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import type { Handoff } from "../handoff/types"; +import { authorize } from "../policy/authorize"; +import { redactText } from "../policy/redact"; +import { DEFAULT_POLICY, type Policy } from "../policy/schema"; +import type { Capability, ExceptionHandler, Step } from "../schema/capability"; +import type { ReplayResult } from "../schema/result"; +import { parseReplayResult } from "../schema/result"; +import { RunSession } from "../session/session"; +import { describeTarget, LocatorError, type Surface } from "../surface/surface"; +import { matchingHandler } from "./handlers"; +import { entryPointOf, resolveValue } from "./values"; + +export type ReplayOptions = { + surface: Surface; + entryPoint?: string; + evidenceDir?: string; + params?: Record; + policy?: Policy; + handoff?: Handoff; + session?: RunSession; +}; + +type Event = ReplayResult["events"][number]; + +export async function replay( + capability: Capability, + options: ReplayOptions, +): Promise { + const params = options.params ?? {}; + const entryPoint = entryPointOf(capability, options.entryPoint); + const policy = options.policy ?? DEFAULT_POLICY; + if (options.session && options.session.surface !== options.surface) { + throw new Error("session.surface must be the replay surface"); + } + const session = options.session ?? new RunSession(options.surface); + const surface = session.surface; + const events: Event[] = []; + const outputs: Record = {}; + + const scrub = (text: string) => + redactText(text, params, capability.parameters); + + const fail = async ( + step: Step, + expected: string, + observed: string, + ): Promise => { + let evidencePath: string | undefined; + if (options.evidenceDir) { + await mkdir(options.evidenceDir, { recursive: true }); + evidencePath = join( + options.evidenceDir, + `${capability.id}-${step.id}.png`, + ); + await surface.screenshot(evidencePath); + } + return parseReplayResult({ + status: "failed", + capabilityId: capability.id, + revision: capability.revision, + stepId: step.id, + expected, + observed: scrub(observed), + evidencePath, + events, + }); + }; + + const escalate = async ( + step: Step, + reason: string, + ): Promise => { + events.push({ + type: "escalated", + stepId: step.id, + detail: reason, + }); + if (!options.handoff) { + return parseReplayResult({ + status: "escalated", + capabilityId: capability.id, + revision: capability.revision, + stepId: step.id, + reason, + events, + }); + } + + session.cede(); + let screenshotPath: string | undefined; + if (options.evidenceDir) { + await mkdir(options.evidenceDir, { recursive: true }); + screenshotPath = join( + options.evidenceDir, + `${capability.id}-${step.id}-hitl.png`, + ); + await surface.screenshot(screenshotPath); + } + const outcome = await options.handoff.intervene(session, { + capabilityId: capability.id, + step, + reason, + observation: await surface.observe(), + screenshotPath, + }); + for (const action of outcome.actions) { + events.push({ + type: "human", + stepId: step.id, + detail: action.detail, + }); + } + session.resume(); + if (session.owner !== "automation") { + throw new Error("handoff returned without restoring automation control"); + } + if (outcome.resume === "abort") { + return parseReplayResult({ + status: "escalated", + capabilityId: capability.id, + revision: capability.revision, + stepId: step.id, + reason, + events, + }); + } + return outcome.resume; + }; + + let index = 0; + while (index < capability.steps.length) { + const step = capability.steps[index]; + if (!step) break; + + let resolvedUrl: string | undefined; + if (step.action === "navigate") { + try { + resolvedUrl = resolveValue(step.url, params, entryPoint); + } catch (error) { + if ( + error instanceof Error && + error.message.startsWith("missing parameter") + ) { + return fail(step, error.message, "parameter not supplied"); + } + throw error; + } + } + + const denial = authorize(step, resolvedUrl, policy); + if (denial?.kind === "escalate") { + const next = await escalate(step, denial.reason); + if (next === "skip_step") { + index += 1; + continue; + } + if (next === "retry_step") continue; + return next; + } + if (denial?.kind === "block") { + return fail(step, "policy allowlist", denial.reason); + } + + try { + await runStep(step, { + params, + entryPoint, + surface, + outputs, + resolvedUrl, + }); + } catch (error) { + if (error instanceof LocatorError) { + events.push({ + type: "acted", + stepId: step.id, + detail: "locator miss after retries", + }); + return fail(step, error.expected, error.observed); + } + if ( + error instanceof Error && + error.message.startsWith("missing parameter") + ) { + return fail(step, error.message, "parameter not supplied"); + } + throw error; + } + + events.push({ + type: "acted", + stepId: step.id, + detail: describeStep(step), + }); + + const observation = await surface.observe(); + const handler = matchingHandler(step.on, observation); + if (!handler) { + index += 1; + continue; + } + + const handled = await applyHandler(handler, step, { + surface, + events, + }); + if (handled === "continue") { + index += 1; + continue; + } + if (handled.status === "business_outcome") { + return parseReplayResult({ + status: "business_outcome", + capabilityId: capability.id, + revision: capability.revision, + code: handled.code, + events, + }); + } + if (handled.status === "escalated") { + const next = await escalate(step, handled.reason); + if (next === "skip_step") { + index += 1; + continue; + } + if (next === "retry_step") continue; + return next; + } + return fail(step, handled.expected, handled.observed); + } + + const checkpoint = capability.success.checkpoint; + if (!(await surface.checkpointMet(checkpoint))) { + const last = capability.steps.at(-1); + const observed = (await surface.observe()).text.slice(0, 240); + events.push({ + type: "checkpoint", + detail: `failed ${checkpoint.kind} "${checkpoint.value}"`, + }); + if (!last) { + throw new Error("capability has no steps"); + } + return fail( + last, + `${checkpoint.kind} "${checkpoint.value}"`, + observed || "(empty page)", + ); + } + + events.push({ + type: "checkpoint", + detail: `${checkpoint.kind} "${checkpoint.value}"`, + }); + + return parseReplayResult({ + status: "success", + capabilityId: capability.id, + revision: capability.revision, + outputs, + events, + }); +} + +async function runStep( + step: Step, + ctx: { + params: Record; + entryPoint: string; + surface: Surface; + outputs: Record; + resolvedUrl?: string; + }, +) { + switch (step.action) { + case "navigate": + await ctx.surface.goto(ctx.resolvedUrl ?? ctx.entryPoint); + return; + case "click": + await ctx.surface.click(step.target); + return; + case "fill": + await ctx.surface.fill( + step.target, + resolveValue(step.value, ctx.params, ctx.entryPoint), + ); + return; + case "select": + await ctx.surface.select( + step.target, + resolveValue(step.value, ctx.params, ctx.entryPoint), + ); + return; + case "extract": + ctx.outputs[step.output] = await ctx.surface.extract(step.target); + return; + case "assert": + if (!(await ctx.surface.checkpointMet(step.checkpoint))) { + throw new LocatorError( + `${step.checkpoint.kind} "${step.checkpoint.value}"`, + (await ctx.surface.observe()).text.slice(0, 240), + ); + } + return; + case "dismiss": + await ctx.surface.dismiss(step.target); + return; + case "wait": + if (!(await ctx.surface.checkpointMet(step.until))) { + throw new LocatorError( + `${step.until.kind} "${step.until.value}"`, + (await ctx.surface.observe()).text.slice(0, 240), + ); + } + } +} + +function describeStep(step: Step): string { + switch (step.action) { + case "navigate": + return "navigate"; + case "click": + return `click ${describeTarget(step.target)}`; + case "fill": + return step.value.kind === "param" + ? `fill from param.${step.value.name}` + : "fill"; + case "select": + return "select"; + case "extract": + return `extract ${step.output}`; + case "assert": + return `assert ${step.checkpoint.kind}`; + case "dismiss": + return "dismiss"; + case "wait": + return "wait"; + } +} + +type HandlerResult = + | "continue" + | { status: "business_outcome"; code: string } + | { status: "escalated"; reason: string } + | { status: "failed"; expected: string; observed: string }; + +async function applyHandler( + handler: ExceptionHandler, + step: Step, + ctx: { surface: Surface; events: Event[] }, +): Promise { + const then = handler.then; + switch (then.type) { + case "business_outcome": + return { status: "business_outcome", code: then.code }; + case "escalate": + return { status: "escalated", reason: then.reason }; + case "fail": + return { + status: "failed", + expected: then.reason, + observed: (await ctx.surface.observe()).text.slice(0, 240), + }; + case "recover": { + const attempts = then.maxAttempts ?? 1; + for (let i = 0; i < attempts; i += 1) { + if (then.action === "dismiss" && "target" in step) { + await ctx.surface.dismiss(step.target); + } else if (then.action === "wait") { + await new Promise((resolve) => setTimeout(resolve, 250)); + } + ctx.events.push({ + type: "recovered", + stepId: step.id, + detail: `${then.action} (${i + 1}/${attempts})`, + }); + const again = matchingHandler(step.on, await ctx.surface.observe()); + if (!again) return "continue"; + } + return { + status: "failed", + expected: `recover via ${then.action}`, + observed: (await ctx.surface.observe()).text.slice(0, 240), + }; + } + } +} diff --git a/src/replay/handlers.ts b/src/replay/handlers.ts new file mode 100644 index 0000000..347393c --- /dev/null +++ b/src/replay/handlers.ts @@ -0,0 +1,23 @@ +import type { ExceptionHandler, ObservationMatch } from "../schema/capability"; +import type { Observation } from "../surface/surface"; + +export function matchingHandler( + handlers: ExceptionHandler[] | undefined, + observation: Observation, +): ExceptionHandler | undefined { + if (!handlers) return undefined; + return handlers.find((handler) => matches(handler.match, observation)); +} + +function matches(match: ObservationMatch, observation: Observation): boolean { + switch (match.kind) { + case "text": + return observation.text.includes(match.value); + case "url": + return observation.url.includes(match.value); + case "dialog": + return (observation.dialog ?? "").includes(match.value); + case "role_name": + return observation.text.includes(match.value); + } +} diff --git a/src/replay/values.ts b/src/replay/values.ts new file mode 100644 index 0000000..04ab232 --- /dev/null +++ b/src/replay/values.ts @@ -0,0 +1,28 @@ +import type { Capability, ValueRef } from "../schema/capability"; + +export function resolveValue( + ref: ValueRef, + params: Record, + entryPoint: string, +): string { + switch (ref.kind) { + case "literal": + return ref.value; + case "entry": + return entryPoint; + case "param": { + const value = params[ref.name]; + if (value === undefined) { + throw new Error(`missing parameter "${ref.name}"`); + } + return value; + } + } +} + +export function entryPointOf( + capability: Capability, + override?: string, +): string { + return override ?? capability.app.entryPoint; +} diff --git a/src/schema/capability.ts b/src/schema/capability.ts index 332ffef..d3be14f 100644 --- a/src/schema/capability.ts +++ b/src/schema/capability.ts @@ -11,7 +11,12 @@ export const SensitivitySchema = z.enum([ export type Sensitivity = z.infer; export const ParamTypeSchema = z.enum(["string", "number", "boolean"]); -export const OutputTypeSchema = z.enum(["string", "number", "boolean", "money"]); +export const OutputTypeSchema = z.enum([ + "string", + "number", + "boolean", + "money", +]); export const ParameterSchema = z.strictObject({ name: z.string().min(1), @@ -109,6 +114,7 @@ export type ExceptionThen = z.infer; export const ExceptionHandlerSchema = z.strictObject({ match: ObservationMatchSchema, + // biome-ignore lint/suspicious/noThenProperty: handler verb in the artifact, not a thenable then: ExceptionThenSchema, }); export type ExceptionHandler = z.infer; @@ -218,7 +224,11 @@ export const CapabilitySchema = z } for (const [index, step] of capability.steps.entries()) { - if ("value" in step && step.value.kind === "param" && !paramNames.has(step.value.name)) { + if ( + "value" in step && + step.value.kind === "param" && + !paramNames.has(step.value.name) + ) { ctx.addIssue({ code: "custom", message: `step value refers to unknown parameter "${step.value.name}"`, diff --git a/src/schema/index.ts b/src/schema/index.ts index 734d279..06b78c7 100644 --- a/src/schema/index.ts +++ b/src/schema/index.ts @@ -1,19 +1,19 @@ export { - SCHEMA_VERSION, - CapabilitySchema, - parseCapability, type Capability, - type Step, + CapabilitySchema, type Locator, type LocatorSet, - type Parameter, type OutputField, + type Parameter, + parseCapability, type Risk, + SCHEMA_VERSION, type Sensitivity, + type Step, } from "./capability"; export { - ReplayResultSchema, parseReplayResult, type ReplayResult, + ReplayResultSchema, } from "./result"; diff --git a/src/schema/result.ts b/src/schema/result.ts index e97ca8b..3b9bbbc 100644 --- a/src/schema/result.ts +++ b/src/schema/result.ts @@ -1,7 +1,7 @@ import { z } from "zod"; const EventSchema = z.strictObject({ - type: z.enum(["acted", "recovered", "checkpoint", "escalated"]), + type: z.enum(["acted", "recovered", "checkpoint", "escalated", "human"]), stepId: z.string().min(1).optional(), detail: z.string().min(1), }); diff --git a/src/session/session.ts b/src/session/session.ts new file mode 100644 index 0000000..c2fef49 --- /dev/null +++ b/src/session/session.ts @@ -0,0 +1,18 @@ +import type { Surface } from "../surface/surface"; + +export type Owner = "automation" | "human"; + +/** Who may act on the live surface. Handoff flips this; it does not open a new session. */ +export class RunSession { + owner: Owner = "automation"; + + constructor(readonly surface: Surface) {} + + cede() { + this.owner = "human"; + } + + resume() { + this.owner = "automation"; + } +} diff --git a/src/surface/playwright-web.ts b/src/surface/playwright-web.ts new file mode 100644 index 0000000..2708b8f --- /dev/null +++ b/src/surface/playwright-web.ts @@ -0,0 +1,169 @@ +import { + type Browser, + chromium, + type Page, + type Locator as PwLocator, +} from "playwright"; +import type { Checkpoint, Locator, LocatorSet } from "../schema/capability"; +import { + describeTarget, + LocatorError, + type Observation, + type Surface, +} from "./surface"; + +const TIMEOUT_MS = 5_000; + +export class PlaywrightWebSurface implements Surface { + private constructor( + private readonly browser: Browser, + private readonly page: Page, + ) {} + + static async launch( + options: { headless?: boolean } = {}, + ): Promise { + const browser = await chromium.launch({ + headless: options.headless ?? true, + }); + const page = await browser.newPage(); + page.setDefaultTimeout(TIMEOUT_MS); + return new PlaywrightWebSurface(browser, page); + } + + async goto(url: string) { + await this.page.goto(url, { waitUntil: "domcontentloaded" }); + } + + async click(target: LocatorSet) { + await (await this.resolve(target)).click(); + } + + async fill(target: LocatorSet, value: string) { + await (await this.resolve(target)).fill(value); + } + + async select(target: LocatorSet, value: string) { + await (await this.resolve(target)).selectOption(value); + } + + async extract(target: LocatorSet) { + return (await (await this.resolve(target)).innerText()).trim(); + } + + async dismiss(target: LocatorSet) { + await (await this.resolve(target)).click(); + } + + async observe(): Promise { + const dialog = this.page.locator('[role="dialog"], [role="alertdialog"]'); + const dialogText = + (await dialog.count()) > 0 + ? (await dialog.first().innerText()).trim() + : undefined; + return { + url: this.page.url(), + text: await this.page.locator("body").innerText(), + dialog: dialogText, + }; + } + + async checkpointMet(checkpoint: Checkpoint) { + switch (checkpoint.kind) { + case "text": + return (await this.observe()).text.includes(checkpoint.value); + case "url": + return this.page.url().includes(checkpoint.value); + case "heading": + return ( + (await this.page + .getByRole("heading", { name: checkpoint.value }) + .count()) > 0 + ); + case "role_name": { + const [role, name] = checkpoint.value.split(":", 2); + if (!role || !name) return false; + return (await this.page.getByRole(asRole(role), { name }).count()) > 0; + } + } + } + + async screenshot(path: string) { + await this.page.screenshot({ path, fullPage: true }); + } + + async close() { + await this.browser.close(); + } + + private async resolve(target: LocatorSet): Promise { + for (const candidate of target.candidates) { + const locator = await this.candidateLocator(candidate); + if (!locator) continue; + if ((await locator.count()) > 0 && (await locator.first().isVisible())) { + return locator.first(); + } + } + const observed = (await this.observe()).text.slice(0, 240); + throw new LocatorError(describeTarget(target), observed || "(empty page)"); + } + + private async candidateLocator( + candidate: Locator, + ): Promise { + switch (candidate.strategy) { + case "role_name": + return this.page.getByRole(asRole(candidate.role), { + name: candidate.name, + exact: candidate.exact, + }); + case "label": + return this.page.getByLabel(candidate.label); + case "nearby_text": + return candidate.role + ? this.page.getByRole(asRole(candidate.role), { + name: candidate.text, + }) + : this.page.getByText(candidate.text); + case "table_cell": + return this.tableCell(candidate.rowText, candidate.columnHeader); + case "structural": + return this.page.locator(candidate.path); + case "css": + return this.page.locator(candidate.selector); + } + } + + private async tableCell( + rowText: string, + columnHeader: string, + ): Promise { + const tables = this.page.locator("table"); + const tableCount = await tables.count(); + for (let i = 0; i < tableCount; i += 1) { + const table = tables.nth(i); + const rows = table.locator(":scope > tbody > tr, :scope > tr"); + const rowCount = await rows.count(); + if (rowCount === 0) continue; + const headers = ( + await rows.first().locator(":scope > th").allTextContents() + ).map((h) => h.trim()); + const col = headers.indexOf(columnHeader); + if (col < 0) continue; + for (let r = 1; r < rowCount; r += 1) { + const row = rows.nth(r); + const cells = (await row.locator(":scope > td").allTextContents()).map( + (c) => c.trim(), + ); + if (cells.includes(rowText) && cells[col]) { + return row.locator(":scope > td").nth(col); + } + } + } + return null; + } +} + +function asRole(role: string): Parameters[0] { + return role as Parameters[0]; +} diff --git a/src/surface/surface.ts b/src/surface/surface.ts new file mode 100644 index 0000000..fb6462e --- /dev/null +++ b/src/surface/surface.ts @@ -0,0 +1,55 @@ +import type { Checkpoint, LocatorSet } from "../schema/capability"; + +export type Observation = { + url: string; + text: string; + dialog?: string; +}; + +export type Surface = { + goto(url: string): Promise; + click(target: LocatorSet): Promise; + fill(target: LocatorSet, value: string): Promise; + select(target: LocatorSet, value: string): Promise; + extract(target: LocatorSet): Promise; + dismiss(target: LocatorSet): Promise; + observe(): Promise; + checkpointMet(checkpoint: Checkpoint): Promise; + screenshot(path: string): Promise; + close(): Promise; +}; + +export class LocatorError extends Error { + constructor( + readonly expected: string, + readonly observed: string, + ) { + super(`locator miss: expected ${expected}; observed ${observed}`); + this.name = "LocatorError"; + } +} + +export function describeTarget(target: LocatorSet): string { + return target.candidates + .map((candidate) => { + switch (candidate.strategy) { + case "role_name": + return `${candidate.role} "${candidate.name}"`; + case "label": + return `label "${candidate.label}"`; + case "nearby_text": + return `near "${candidate.text}"`; + case "table_cell": + return `table[${candidate.rowText} / ${candidate.columnHeader}]`; + case "structural": + return candidate.path; + case "css": + return candidate.selector; + default: { + const _exhaustive: never = candidate; + return _exhaustive; + } + } + }) + .join(" | "); +} diff --git a/tests/app/core-servicing.test.ts b/tests/app/core-servicing.test.ts new file mode 100644 index 0000000..54465e8 --- /dev/null +++ b/tests/app/core-servicing.test.ts @@ -0,0 +1,68 @@ +import type { Server } from "node:http"; +import { afterEach, describe, expect, it } from "vitest"; +import { createServer } from "../../apps/core-servicing/server"; + +const servers: Server[] = []; + +async function start() { + const server = createServer(); + servers.push(server); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("expected a TCP address"); + } + return `http://127.0.0.1:${address.port}`; +} + +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }), + ), + ); +}); + +async function page(base: string, path = "/") { + return (await fetch(`${base}${path}`)).text(); +} + +describe("core-servicing app", () => { + it("serves a search form with Member ID and Search, and no test ids", async () => { + const html = await page(await start()); + + expect(html).toContain('label for="f1">Member ID'); + expect(html).toContain('type="submit" id="cmd1" value="Search"'); + expect(html).not.toMatch(/data-testid/); + expect(html).not.toMatch(/id="member-/i); + }); + + it("shows member detail and the savings balance for a known member", async () => { + const html = await page(await start(), "/member?id=10001"); + + expect(html).toContain("

Member detail

"); + expect(html).toContain("AccountBalance"); + expect(html).toContain("Savings"); + expect(html).toContain("$1,240.50"); + expect(html).not.toContain("Member not found"); + }); + + it("shows Member not found for an unknown id", async () => { + const html = await page(await start(), "/member?id=99999"); + + expect(html).toContain("Member not found"); + expect(html).not.toContain("

Member detail

"); + }); + + it("shows a validation message when member id is missing", async () => { + const html = await page(await start(), "/member?id="); + + expect(html).toContain("Member ID is required"); + expect(html).not.toContain("

Member detail

"); + }); +}); diff --git a/tests/handoff/hitl.test.ts b/tests/handoff/hitl.test.ts new file mode 100644 index 0000000..68e94cd --- /dev/null +++ b/tests/handoff/hitl.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from "vitest"; +import { ScriptedOperator } from "../../src/handoff/scripted"; +import { replay } from "../../src/replay/engine"; +import { parseCapability } from "../../src/schema/capability"; +import { RunSession } from "../../src/session/session"; +import { RecordingSurface } from "../helpers/recording-surface"; + +const clickTarget = { + candidates: [{ strategy: "role_name" as const, role: "button", name: "Go" }], +}; + +function capability(overrides: Record = {}) { + return parseCapability({ + schemaVersion: "1.0.0", + id: "hitl-probe", + name: "HITL probe", + description: "Tiny flow for handoff tests", + revision: 1, + app: { + family: "core-servicing", + surface: "web", + entryPoint: "http://127.0.0.1:4173/", + }, + parameters: [], + outputs: [ + { + name: "savingsBalance", + type: "money", + sensitivity: "financial", + description: "Balance after the human finishes the blocked step", + }, + ], + steps: [ + { + id: "open-app", + action: "navigate", + url: { kind: "entry" }, + risk: "read", + }, + ], + success: { checkpoint: { kind: "text", value: "ok" } }, + ...overrides, + }); +} + +describe("HITL handoff", () => { + it("lets a human act on the same surface, then continues remaining steps", async () => { + const surface = new RecordingSurface(); + const session = new RunSession(surface); + let ownerDuringIntervene: string | undefined; + + const result = await replay( + capability({ + steps: [ + { + id: "open-app", + action: "navigate", + url: { kind: "entry" }, + risk: "read", + }, + { + id: "submit", + action: "click", + target: clickTarget, + risk: "irreversible", + }, + { + id: "extract-balance", + action: "extract", + target: { + candidates: [{ strategy: "css", selector: "td" }], + }, + output: "savingsBalance", + risk: "read", + }, + ], + }), + { + surface, + session, + handoff: new ScriptedOperator(async (owned, request) => { + ownerDuringIntervene = owned.owner; + expect(owned.surface).toBe(surface); + if (request.step.action !== "click") { + throw new Error("expected click step"); + } + await owned.surface.click(request.step.target); + return { + actions: [{ type: "click", detail: "clicked Go" }], + resume: "skip_step", + }; + }), + }, + ); + + expect(ownerDuringIntervene).toBe("human"); + expect(session.owner).toBe("automation"); + expect(result.status).toBe("success"); + expect(surface.clicks).toBe(1); + expect(surface.gotos).toEqual(["http://127.0.0.1:4173/"]); + expect( + result.events.some( + (event) => event.type === "human" && event.detail === "clicked Go", + ), + ).toBe(true); + expect( + result.events.some( + (event) => event.type === "acted" && event.stepId === "extract-balance", + ), + ).toBe(true); + }); + + it("aborts as escalated and restores automation control", async () => { + const surface = new RecordingSurface(); + const session = new RunSession(surface); + const result = await replay( + capability({ + steps: [ + { + id: "submit", + action: "click", + target: clickTarget, + risk: "irreversible", + }, + ], + }), + { + surface, + session, + handoff: new ScriptedOperator(async (owned) => { + expect(owned.owner).toBe("human"); + return { + actions: [{ type: "other", detail: "operator aborted" }], + resume: "abort", + }; + }), + }, + ); + + expect(result.status).toBe("escalated"); + expect(session.owner).toBe("automation"); + expect(surface.clicks).toBe(0); + if (result.status === "escalated") { + expect(result.reason).toContain("irreversible"); + } + expect(result.events.some((event) => event.type === "human")).toBe(true); + }); +}); diff --git a/tests/helpers/http-app.ts b/tests/helpers/http-app.ts new file mode 100644 index 0000000..6d1793b --- /dev/null +++ b/tests/helpers/http-app.ts @@ -0,0 +1,23 @@ +import type { Server } from "node:http"; +import { createServer } from "../../apps/core-servicing/server"; + +export async function startApp(): Promise<{ + baseUrl: string; + close: () => Promise; +}> { + const server: Server = createServer(); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("expected a TCP address"); + } + return { + baseUrl: `http://127.0.0.1:${address.port}`, + close: () => + new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }), + }; +} diff --git a/tests/helpers/recording-surface.ts b/tests/helpers/recording-surface.ts new file mode 100644 index 0000000..78ae302 --- /dev/null +++ b/tests/helpers/recording-surface.ts @@ -0,0 +1,37 @@ +import type { Checkpoint, LocatorSet } from "../../src/schema/capability"; +import type { Observation, Surface } from "../../src/surface/surface"; + +export class RecordingSurface implements Surface { + readonly gotos: string[] = []; + clicks = 0; + + async goto(url: string) { + this.gotos.push(url); + } + + async click(_target: LocatorSet) { + this.clicks += 1; + } + + async fill(_target: LocatorSet, _value: string) {} + + async select(_target: LocatorSet, _value: string) {} + + async extract(_target: LocatorSet) { + return ""; + } + + async dismiss(_target: LocatorSet) {} + + async observe(): Promise { + return { url: this.gotos.at(-1) ?? "", text: "" }; + } + + async checkpointMet(_checkpoint: Checkpoint) { + return true; + } + + async screenshot(_path: string) {} + + async close() {} +} diff --git a/tests/policy/guard.test.ts b/tests/policy/guard.test.ts new file mode 100644 index 0000000..8df2c4f --- /dev/null +++ b/tests/policy/guard.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "vitest"; +import { authorizeUrl } from "../../src/policy/authorize"; +import { redactText } from "../../src/policy/redact"; +import { DEFAULT_POLICY } from "../../src/policy/schema"; +import { replay } from "../../src/replay/engine"; +import { parseCapability } from "../../src/schema/capability"; +import { RecordingSurface } from "../helpers/recording-surface"; + +const clickTarget = { + candidates: [{ strategy: "role_name" as const, role: "button", name: "Go" }], +}; + +function capability(overrides: Record = {}) { + return parseCapability({ + schemaVersion: "1.0.0", + id: "policy-probe", + name: "Policy probe", + description: "Tiny flow for policy tests", + revision: 1, + app: { + family: "core-servicing", + surface: "web", + entryPoint: "http://127.0.0.1:4173/", + }, + parameters: [ + { + name: "memberId", + type: "string", + sensitivity: "identifier", + description: "Member number", + }, + ], + outputs: [], + steps: [ + { + id: "open-app", + action: "navigate", + url: { kind: "entry" }, + risk: "read", + }, + ], + success: { checkpoint: { kind: "text", value: "ok" } }, + ...overrides, + }); +} + +describe("redactText", () => { + it("replaces identifier param values and leaves other text", () => { + expect( + redactText("member 10001 not found", { memberId: "10001" }, [ + { + name: "memberId", + type: "string", + sensitivity: "identifier", + description: "Member number", + }, + ]), + ).toBe("member [memberId] not found"); + }); +}); + +describe("authorizeUrl", () => { + it("allows the local mock host", () => { + expect( + authorizeUrl("http://127.0.0.1:4173/member?id=1", DEFAULT_POLICY), + ).toBeNull(); + }); + + it("blocks a host outside the allowlist", () => { + const denial = authorizeUrl("https://evil.example/steal", DEFAULT_POLICY); + expect(denial?.kind).toBe("block"); + expect(denial?.reason).toContain("evil.example"); + }); +}); + +describe("replay policy", () => { + it("does not navigate to a disallowed host", async () => { + const surface = new RecordingSurface(); + const result = await replay( + capability({ + app: { + family: "core-servicing", + surface: "web", + entryPoint: "https://evil.example/", + }, + }), + { surface }, + ); + + expect(result.status).toBe("failed"); + expect(surface.gotos).toEqual([]); + if (result.status === "failed") { + expect(result.expected).toBe("policy allowlist"); + expect(result.observed).toContain("evil.example"); + } + }); + + it("escalates an irreversible step and does not click", async () => { + const surface = new RecordingSurface(); + const result = await replay( + capability({ + steps: [ + { + id: "submit-transfer", + action: "click", + target: clickTarget, + risk: "irreversible", + }, + ], + }), + { surface }, + ); + + expect(result.status).toBe("escalated"); + expect(surface.clicks).toBe(0); + if (result.status === "escalated") { + expect(result.reason).toContain("irreversible"); + } + }); + + it("blocks an irreversible step when policy says block", async () => { + const surface = new RecordingSurface(); + const result = await replay( + capability({ + steps: [ + { + id: "submit-transfer", + action: "click", + target: clickTarget, + risk: "irreversible", + }, + ], + }), + { + surface, + policy: { ...DEFAULT_POLICY, onIrreversible: "block" }, + }, + ); + + expect(result.status).toBe("failed"); + expect(surface.clicks).toBe(0); + }); + + it("blocks a path outside the allowlist", async () => { + const surface = new RecordingSurface(); + const result = await replay(capability(), { + surface, + policy: { ...DEFAULT_POLICY, allowedPathPrefixes: ["/member"] }, + }); + + expect(result.status).toBe("failed"); + expect(surface.gotos).toEqual([]); + if (result.status === "failed") { + expect(result.observed).toContain('path "/"'); + } + }); +}); diff --git a/tests/replay/lookup-member.test.ts b/tests/replay/lookup-member.test.ts new file mode 100644 index 0000000..d25ac61 --- /dev/null +++ b/tests/replay/lookup-member.test.ts @@ -0,0 +1,151 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { ScriptedOperator } from "../../src/handoff/scripted"; +import { replay } from "../../src/replay/engine"; +import { parseCapability } from "../../src/schema/capability"; +import { RunSession } from "../../src/session/session"; +import { PlaywrightWebSurface } from "../../src/surface/playwright-web"; +import { startApp } from "../helpers/http-app"; + +const fixture = parseCapability( + JSON.parse( + readFileSync( + join( + dirname(fileURLToPath(import.meta.url)), + "../fixtures/lookup-member-savings.json", + ), + "utf8", + ), + ), +); + +const apps: Array<{ close: () => Promise }> = []; +const surfaces: PlaywrightWebSurface[] = []; + +afterEach(async () => { + await Promise.all(surfaces.splice(0).map((surface) => surface.close())); + await Promise.all(apps.splice(0).map((app) => app.close())); +}); + +describe("replay lookup-member-savings", () => { + it("returns savings balance for member 10001 without calling an LLM", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + + const result = await replay(fixture, { + surface, + entryPoint: `${app.baseUrl}/`, + params: { memberId: "10001" }, + }); + + expect(result.status).toBe("success"); + if (result.status === "success") { + expect(result.outputs.savingsBalance).toBe("$1,240.50"); + } + }, 30_000); + + it("returns member_not_found as a business outcome, not a failure", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + + const result = await replay(fixture, { + surface, + entryPoint: `${app.baseUrl}/`, + params: { memberId: "99999" }, + }); + + expect(result.status).toBe("business_outcome"); + if (result.status === "business_outcome") { + expect(result.code).toBe("member_not_found"); + } + }, 30_000); + + it("fails with step, expected, and observed when a locator misses", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + + const broken = parseCapability({ + ...fixture, + steps: fixture.steps.map((step) => + step.id === "click-search" + ? { + ...step, + target: { + candidates: [ + { + strategy: "role_name", + role: "button", + name: "Definitely not a button", + }, + ], + }, + } + : step, + ), + }); + + const result = await replay(broken, { + surface, + entryPoint: `${app.baseUrl}/`, + params: { memberId: "10001" }, + }); + + expect(result.status).toBe("failed"); + if (result.status === "failed") { + expect(result.stepId).toBe("click-search"); + expect(result.expected).toContain("Definitely not a button"); + expect(result.observed.length).toBeGreaterThan(0); + } + }, 30_000); + + it("hands the live page to a human for an irreversible search, then resumes", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + const session = new RunSession(surface); + + const irreversibleSearch = parseCapability({ + ...fixture, + steps: fixture.steps.map((step) => + step.id === "click-search" ? { ...step, risk: "irreversible" } : step, + ), + }); + + const result = await replay(irreversibleSearch, { + surface, + session, + entryPoint: `${app.baseUrl}/`, + params: { memberId: "10001" }, + handoff: new ScriptedOperator(async (owned, request) => { + expect(owned.owner).toBe("human"); + expect(owned.surface).toBe(surface); + if (request.step.action !== "click") { + throw new Error("expected click step"); + } + await owned.surface.click(request.step.target); + return { + actions: [ + { type: "click", detail: "clicked Search on live session" }, + ], + resume: "skip_step", + }; + }), + }); + + expect(result.status).toBe("success"); + expect(session.owner).toBe("automation"); + if (result.status === "success") { + expect(result.outputs.savingsBalance).toBe("$1,240.50"); + } + expect(result.events.some((event) => event.type === "human")).toBe(true); + }, 30_000); +}); diff --git a/tests/schema/result.test.ts b/tests/schema/result.test.ts index 202d3d3..ade4618 100644 --- a/tests/schema/result.test.ts +++ b/tests/schema/result.test.ts @@ -8,7 +8,13 @@ describe("ReplayResultSchema", () => { capabilityId: "lookup-member-savings", revision: 1, outputs: { savingsBalance: "$1,240.50" }, - events: [{ type: "checkpoint", stepId: "extract-balance", detail: "heading Member detail" }], + events: [ + { + type: "checkpoint", + stepId: "extract-balance", + detail: "heading Member detail", + }, + ], }); expect(result.status).toBe("success"); @@ -47,7 +53,13 @@ describe("ReplayResultSchema", () => { expected: "button Search", observed: "dialog: Session expired", evidencePath: "evidence/replay-failed.png", - events: [{ type: "acted", stepId: "click-search", detail: "locator miss after retries" }], + events: [ + { + type: "acted", + stepId: "click-search", + detail: "locator miss after retries", + }, + ], }); expect(result.status).toBe("failed"); @@ -64,7 +76,13 @@ describe("ReplayResultSchema", () => { revision: 1, stepId: "click-search", reason: "unhandled confirmation dialog", - events: [{ type: "escalated", stepId: "click-search", detail: "control ceded to human" }], + events: [ + { + type: "escalated", + stepId: "click-search", + detail: "control ceded to human", + }, + ], }); expect(result.status).toBe("escalated"); diff --git a/tsconfig.json b/tsconfig.json index 4c5aafd..5ee8744 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,5 +12,5 @@ "noEmit": true, "types": ["node"] }, - "include": ["src", "tests", "vitest.config.ts"] + "include": ["src", "tests", "apps", "vitest.config.ts"] }