feat(studio,mcp,agent): studio_pick, and requests to the page report back (0.6.0)

apps/studio-ui/src/content-pane/content-pane.component.tsx

@@ -5,7 +5,7 @@ import { InspectorPanel, matchingNodeIds } from '../inspector'
 import { allSteps, stepById } from '../steps-outline'
 import type { GridViewOverride } from '../studio-client'
 import { plainError, useDeckViewQuery, useDocumentTreeQuery, useGridViewQuery, useInferSelectorMutation, usePdfViewQuery, useSnapshotQuery, useStartRecordingMutation, useStopRecordingMutation, useStudioClient } from '../studio-client'
-import { useRecordingStore, useStudioUiStore } from '../studio-store'
+import { useRecordingStore, useStudioUiStore, useUiRequestStore } from '../studio-store'
 import { BlankSnapshotNotice } from './blank-snapshot-notice.component'
 import { SnapshotNotice } from './snapshot-notice.component'
 import { TreeViewSwitch } from './tree-view-switch.component'
@@ -19,6 +19,7 @@ import { SelectorOptions } from './selector-options.component'
 import { SnapshotFrame } from './snapshot-frame.component'
 import { pickedSteps } from './step-highlight.mapper'
 import { TreeCanvas } from './tree-canvas.component'
+import { findPickNode } from './ui-pick-target.algorithm'
 import { chosenPick, collectsMany, groupsOf, headingFor, hintFor, manyChoice, stagePick } from './web-pick-staging.mapper'
 import type { StagedWebPick } from './web-pick-staging.mapper'
 import { hasVisibleContent } from './visible-content.algorithm'
@@ -147,6 +148,9 @@ export function ContentPane ({ recipe, onSaveOutline, onSaveRecipe }: ContentPan
   const tableInsertedAtRef = useRef<number | undefined>(undefined)
 
   const pickMode = pickTarget !== undefined && pickTarget.recipeId === recipeId
+  const uiPick = useUiRequestStore(state => state.pick)
+  const finishUiPick = useUiRequestStore(state => state.finishPick)
+  const handledUiPickRef = useRef<string | undefined>(undefined)
 
   // A different recipe means a different (or no) table card to upgrade in place, and a stale CSV override to drop.
   useEffect(() => {
@@ -156,8 +160,8 @@ export function ContentPane ({ recipe, onSaveOutline, onSaveRecipe }: ContentPan
   }, [recipeId])
 
   /** A click on the web page: nothing is written to the recipe yet; the ways to reach the clicked element are shown to choose from, and "Add to recipe" writes it. */
-  async function handlePick (nodeId: string): Promise<void> {
-    if (recipeId === undefined) return
+  async function handlePick (nodeId: string): Promise<StagedWebPick | undefined> {
+    if (recipeId === undefined) return undefined
     // A click on the page before "Read" was pressed starts the pick itself.
     if (!pickMode) {
       insertedAtRef.current = undefined
@@ -170,19 +174,22 @@ export function ContentPane ({ recipe, onSaveOutline, onSaveRecipe }: ContentPan
       if (result.kind !== 'field' && result.kind !== 'item-field') {
         warn(result.kind === 'unsupported' ? result.reason : undefined)
 
-        return
+        return undefined
       }
-      setStagedPick(stagePick(result, clickedText(nodeId)))
+      const staged = stagePick(result, clickedText(nodeId))
+      setStagedPick(staged)
 
-      return
+      return staged
     }
     const result = await inferSelector.mutateAsync({ recipeId, path: STEP_PATH, nodeIds: [outcome.firstNodeId, nodeId] })
     if (result.kind !== 'list') {
       warn(result.kind === 'unsupported' ? result.reason : undefined)
 
-      return
+      return undefined
     }
     setStagedPick(previous => stagePick(result, previous?.text))
+
+    return undefined
   }
 
   /** The text of a node of the web snapshot, for the options panel to show what was picked. */
@@ -196,10 +203,10 @@ export function ContentPane ({ recipe, onSaveOutline, onSaveRecipe }: ContentPan
   }
 
   /** "Add to recipe" for the staged web pick: writes the card(s) with the selectors the person chose. */
-  async function addStagedPick (): Promise<void> {
-    if (recipeId === undefined || stagedPick === undefined) return
-    const result = chosenPick(stagedPick)
-    const many = collectsMany(stagedPick)
+  async function addStagedPick (staged: StagedWebPick | undefined = stagedPick): Promise<void> {
+    if (recipeId === undefined || staged === undefined) return
+    const result = chosenPick(staged)
+    const many = collectsMany(staged)
     setStagedPick(undefined)
     stopPicking()
     if (result.kind === 'item-field') {
@@ -235,6 +242,53 @@ export function ContentPane ({ recipe, onSaveOutline, onSaveRecipe }: ContentPan
     stopPicking()
   }
 
+  // A controller (an agent, through the MCP server) asked to pick a value on this recipe's page: do it as a click would, once the snapshot is
+  // here, and tell the server how it went. Runs on every render but acts once per request.
+  useEffect(() => {
+    const request = uiPick
+    const html = snapshot.data?.html
+    if (!isHtmlSnapshot || html === undefined || request === undefined || request.recipeId !== recipeId) return
+    if (handledUiPickRef.current === request.requestId) return
+    if (pickMode || stagedPick !== undefined) {
+      // A pick of the person's own is under way: clear it, and the next render does the controller's.
+      cancelStagedPick()
+
+      return
+    }
+    handledUiPickRef.current = request.requestId
+    const tell = async (ok: boolean, message: string, detail?: Record<string, unknown>): Promise<void> => {
+      finishUiPick(request.requestId)
+      try {
+        await client.reportUiOutcome(request.requestId, ok, message, detail)
+      } catch {
+        // A controller that never hears back just times out.
+      }
+    }
+    void (async () => {
+      const found = findPickNode(html, request)
+      if ('reason' in found) {
+        await tell(false, `Could not pick: ${found.reason}.`)
+
+        return
+      }
+      const staged = await handlePick(found.nodeId)
+      if (staged === undefined) {
+        await tell(false, 'Studio could not turn that element into a selector.', { text: found.text })
+
+        return
+      }
+      const { field } = chosenPick(staged)
+      const detail = { text: found.text, selector: field.selector, matches: field.matches, alternatives: (field.alternatives ?? []).map(option => ({ selector: option.selector, matches: option.matches })) }
+      if (request.add === true) {
+        await addStagedPick(staged)
+        await tell(true, `Added to the recipe: ${field.selector}`, detail)
+
+        return
+      }
+      await tell(true, 'Staged: the person can choose a selector and press Add to recipe.', detail)
+    })()
+  })
+
   /**
    * Handles a tree canvas pick (studio plan §3.4, issue #94's 5a): `value`
    * and `list` both write a Read card (`documentReadCardNode`, generalised

apps/studio-ui/src/content-pane/ui-pick-target.algorithm.test.ts

@@ -0,0 +1,29 @@
+import { findPickNode } from './ui-pick-target.algorithm'
+
+const PAGE = '<html><body data-oc-node="n0"><h1 data-oc-node="n1">Renault Clio</h1><div data-oc-node="n2" class="price"><span data-oc-node="n3">From</span> <b data-oc-node="n4">€ 19 900</b></div><p data-oc-node="n5">Clio   hatch\n back</p></body></html>'
+
+describe('findPickNode', () => {
+  it('finds an element by its CSS selector', () => {
+    expect(findPickNode(PAGE, { selector: '.price b' })).toEqual({ nodeId: 'n4', text: '€ 19 900' })
+  })
+
+  it('finds the smallest element that shows exactly the text, ignoring case and spacing', () => {
+    expect(findPickNode(PAGE, { text: '€ 19 900' })).toEqual({ nodeId: 'n4', text: '€ 19 900' })
+    expect(findPickNode(PAGE, { text: 'clio hatch back' })).toEqual({ nodeId: 'n5', text: 'Clio hatch back' })
+  })
+
+  it('falls back to the smallest element that contains the text', () => {
+    expect(findPickNode(PAGE, { text: 'Clio' })).toEqual({ nodeId: 'n1', text: 'Renault Clio' })
+  })
+
+  it('prefers the selector when both are given', () => {
+    expect(findPickNode(PAGE, { selector: 'h1', text: '€ 19 900' })).toEqual({ nodeId: 'n1', text: 'Renault Clio' })
+  })
+
+  it('says why nothing was found', () => {
+    expect(findPickNode(PAGE, { selector: '.nope' })).toEqual({ reason: 'nothing on the page matches .nope' })
+    expect(findPickNode(PAGE, { selector: 'div[' })).toEqual({ reason: 'div[ is not a valid CSS selector' })
+    expect(findPickNode(PAGE, { text: 'Megane' })).toEqual({ reason: 'no element on the page shows "Megane"' })
+    expect(findPickNode(PAGE, {})).toEqual({ reason: 'say what to pick: a CSS selector or the text it shows' })
+  })
+})

apps/studio-ui/src/content-pane/ui-pick-target.algorithm.ts

@@ -0,0 +1,60 @@
+/** What a controller names to pick on the web page: a CSS selector, or the text the element shows. */
+export interface UiPickTarget {
+  selector?: string
+  text?:     string
+}
+
+/** The element found (its `data-oc-node` id and the text it shows), or why none was. */
+export type FoundPickNode = { nodeId: string, text: string } | { reason: string }
+
+/** A node's text as a person reads it: whitespace collapsed. */
+function textOf (element: Element): string {
+  return (element.textContent ?? '').replace(/\s+/g, ' ').trim() // eslint-disable-line unicorn/prefer-string-replace-all -- this app's `lib` has no `replaceAll`
+}
+
+function foundOf (element: Element | null): FoundPickNode {
+  const node = element?.closest('[data-oc-node]')
+  const nodeId = node instanceof HTMLElement ? node.dataset.ocNode : undefined
+  if (node === null || node === undefined || nodeId === undefined) return { reason: 'that element is not part of the page snapshot' }
+
+  return { nodeId, text: textOf(node).slice(0, 160) }
+}
+
+/** The element whose own text is the smallest one that equals `text` (case-insensitive), else the smallest one that contains it. */
+function elementShowing (document: Document, text: string): Element | null {
+  const wanted = text.replace(/\s+/g, ' ').trim().toLowerCase() // eslint-disable-line unicorn/prefer-string-replace-all -- this app's `lib` has no `replaceAll`
+  let exact: { element: Element, length: number } | undefined
+  let partial: { element: Element, length: number } | undefined
+  for (const element of document.body.querySelectorAll('[data-oc-node]')) {
+    const shown = textOf(element)
+    const lowered = shown.toLowerCase()
+    if (lowered === wanted && (exact === undefined || shown.length < exact.length)) exact = { element, length: shown.length }
+    else if (lowered.includes(wanted) && (partial === undefined || shown.length < partial.length)) partial = { element, length: shown.length }
+  }
+
+  return exact?.element ?? partial?.element ?? null
+}
+
+/**
+ * Finds the element of a web snapshot a controller asked to pick, the way a person would point at it: by CSS selector, or by the text it shows.
+ *
+ * @param html - The snapshot's HTML (every element carries a `data-oc-node` id).
+ * @param target - The selector and/or text to look for; the selector wins when both are given.
+ * @returns The element's node id and text, or why none was found.
+ */
+export function findPickNode (html: string, target: UiPickTarget): FoundPickNode {
+  const document = new DOMParser().parseFromString(html, 'text/html')
+  if (target.selector !== undefined) {
+    try {
+      const element = document.querySelector(target.selector)
+
+      return element === null ? { reason: `nothing on the page matches ${target.selector}` } : foundOf(element)
+    } catch {
+      return { reason: `${target.selector} is not a valid CSS selector` }
+    }
+  }
+  if (target.text === undefined) return { reason: 'say what to pick: a CSS selector or the text it shows' }
+  const element = elementShowing(document, target.text)
+
+  return element === null ? { reason: `no element on the page shows "${target.text}"` } : foundOf(element)
+}

apps/studio-ui/src/recipe-start/ui-requests.hook.test.tsx

@@ -2,7 +2,7 @@ import { act, renderHook, waitFor } from '@testing-library/react'
 import { QueryClientProvider } from '@tanstack/react-query'
 import type { PropsWithChildren } from 'react'
 import { createStudioQueryClient } from '../studio-client'
-import { resetStudioUiStore, useStudioUiStore } from '../studio-store'
+import { resetStudioUiStore, resetUiRequestStore, useStudioUiStore, useUiRequestStore } from '../studio-store'
 import { useUiRequests } from './ui-requests.hook'
 
 let messageHandler: ((event: { data: string }) => void) | undefined
@@ -15,9 +15,11 @@ class FakeWebSocket {
   close (): void {}
 }
 
-function mockFetch (answer: (command: { type: string, path?: string }) => { ok: boolean, body: unknown }): jest.Mock {
+interface Sent { type: string, path?: string, requestId?: string, ok?: boolean, message?: string }
+
+function mockFetch (answer: (command: Sent) => { ok: boolean, body: unknown }): jest.Mock {
   const fetchMock = jest.fn().mockImplementation(async (_url: string, init: RequestInit) => {
-    const { ok, body } = answer(JSON.parse(init.body as string) as { type: string, path?: string })
+    const { ok, body } = answer(JSON.parse(init.body as string) as Sent)
 
     return { ok, status: ok ? 200 : 500, json: async () => body }
   })
@@ -26,6 +28,14 @@ function mockFetch (answer: (command: { type: string, path?: string }) => { ok:
   return fetchMock
 }
 
+function sentTypes (fetchMock: jest.Mock): string[] {
+  return fetchMock.mock.calls.map(([, init]) => (JSON.parse((init as RequestInit).body as string) as Sent).type)
+}
+
+function reports (fetchMock: jest.Mock): Sent[] {
+  return fetchMock.mock.calls.map(([, init]) => JSON.parse((init as RequestInit).body as string) as Sent).filter(command => command.type === 'ui-report')
+}
+
 function wrapper ({ children }: PropsWithChildren): React.ReactElement {
   return <QueryClientProvider client={createStudioQueryClient()}>{children}</QueryClientProvider>
 }
@@ -34,28 +44,43 @@ beforeAll(() => { Object.defineProperty(globalThis, 'WebSocket', { value: FakeWe
 beforeEach(() => {
   history.replaceState({}, '', '/?token=abc123')
   resetStudioUiStore()
+  resetUiRequestStore()
   messageHandler = undefined
 })
 
 describe('useUiRequests', () => {
   it('opens the recipe a controller asked for, the way a link does', async () => {
-    mockFetch(command => ({ ok: true, body: command.type === 'open-path' ? { kind: 'recipe', folder: '/work', recipeId: 'books' } : { folder: '/work', recipes: [] } }))
+    const fetchMock = mockFetch(command => ({ ok: true, body: command.type === 'open-path' ? { kind: 'recipe', folder: '/work', recipeId: 'books' } : (command.type === 'ui-report' ? { kept: true } : { folder: '/work', recipes: [] }) }))
     renderHook(() => useUiRequests(), { wrapper })
 
-    act(() => { messageHandler?.({ data: JSON.stringify({ type: 'ui-request', request: { kind: 'open', path: '/work/books.input.json' } }) }) })
+    act(() => { messageHandler?.({ data: JSON.stringify({ type: 'ui-request', requestId: 'r1', request: { kind: 'open', path: '/work/books.input.json' } }) }) })
 
     await waitFor(() => { expect(useStudioUiStore.getState().selectedRecipeId).toBe('books') })
     expect(useStudioUiStore.getState().openFolder).toBe('/work')
+    await waitFor(() => { expect(sentTypes(fetchMock)).toContain('ui-report') })
+    expect(reports(fetchMock)).toEqual([{ type: 'ui-report', requestId: 'r1', ok: true, message: 'Opened /work/books.input.json.' }])
   })
 
   it('shows the reason when the path cannot be opened, and ignores other events', async () => {
-    mockFetch(() => ({ ok: false, body: { error: 'no such file' } }))
+    const fetchMock = mockFetch(command => (command.type === 'ui-report' ? { ok: true, body: { kept: true } } : { ok: false, body: { error: 'no such file' } }))
     const { result } = renderHook(() => useUiRequests(), { wrapper })
 
     act(() => { messageHandler?.({ data: JSON.stringify({ type: 'workspace-changed' }) }) })
     expect(result.current).toBeUndefined()
-    act(() => { messageHandler?.({ data: JSON.stringify({ type: 'ui-request', request: { kind: 'open', path: '/nope' } }) }) })
+    act(() => { messageHandler?.({ data: JSON.stringify({ type: 'ui-request', requestId: 'r2', request: { kind: 'open', path: '/nope' } }) }) })
 
     await waitFor(() => { expect(result.current).toBeDefined() })
+    await waitFor(() => { expect(reports(fetchMock)).toHaveLength(1) })
+    expect(reports(fetchMock)[0]).toMatchObject({ requestId: 'r2', ok: false })
+  })
+
+  it('selects the recipe a pick is for and leaves the pick to the content pane', () => {
+    mockFetch(() => ({ ok: true, body: {} }))
+    renderHook(() => useUiRequests(), { wrapper })
+
+    act(() => { messageHandler?.({ data: JSON.stringify({ type: 'ui-request', requestId: 'r3', request: { kind: 'pick', recipeId: 'cars', text: 'Clio', add: true } }) }) })
+
+    expect(useStudioUiStore.getState().selectedRecipeId).toBe('cars')
+    expect(useUiRequestStore.getState().pick).toEqual({ requestId: 'r3', recipeId: 'cars', selector: undefined, text: 'Clio', add: true })
   })
 })

apps/studio-ui/src/recipe-start/ui-requests.hook.ts

@@ -1,25 +1,54 @@
 import { useEffect, useState } from 'react'
 import { useStudioClient } from '../studio-client'
+import { useStudioUiStore, useUiRequestStore } from '../studio-store'
 import { useOpenTarget } from './open-path.hook'
 
+function messageOf (error: unknown): string {
+  return error instanceof Error ? error.message : String(error)
+}
+
 /**
- * Does what a controller asked this page to do (an agent, through the MCP server, so the person watching sees it happen):
- * today, to open a folder, a recipe or a document the way a Studio link does. Call once, near the app's root.
+ * Does what a controller asked this page to do (an agent, through the MCP server, so the person watching sees it happen)
+ * and tells the server how it went, so the controller can find out: open a folder, a recipe or a document the way a Studio
+ * link does, or pick a value on a recipe's web page (the recipe is selected here; the content pane does the pick once
+ * its snapshot is there, and reports). Call once, near the app's root.
  *
  * @returns The error to show when a request could not be done.
  */
 export function useUiRequests (): string | undefined {
   const client = useStudioClient()
   const openTarget = useOpenTarget()
+  const selectRecipe = useStudioUiStore(state => state.selectRecipe)
+  const askPick = useUiRequestStore(state => state.askPick)
   const [error, setError] = useState<string>()
 
   useEffect(() => client.subscribe((event) => {
     if (event.type !== 'ui-request') return
+    const { requestId, request } = event
     setError(undefined)
-    void openTarget(event.request.path, event.request.workspace).catch((openError: unknown) => {
-      setError(openError instanceof Error ? openError.message : String(openError))
-    })
-  }), [client, openTarget])
+    if (request.kind === 'pick') {
+      selectRecipe(request.recipeId)
+      askPick({ requestId, recipeId: request.recipeId, selector: request.selector, text: request.text, add: request.add })
+
+      return
+    }
+    void (async () => {
+      let ok = true
+      let message = `Opened ${request.path}.`
+      try {
+        await openTarget(request.path, request.workspace)
+      } catch (openError) {
+        ok = false
+        message = messageOf(openError)
+        setError(message)
+      }
+      try {
+        await client.reportUiOutcome(requestId, ok, message)
+      } catch {
+        // A controller that never hears back just times out.
+      }
+    })()
+  }), [client, openTarget, selectRecipe, askPick])
 
   return error
 }