feat(mcp): a docs tool for agents (#292)

.gitignore

@@ -78,3 +78,6 @@ apps/vscode-extension/*.vsix
 
 # Field-test inputs and results (third-party documents, kept on this machine only)
 /studio-field-tests/
+
+# The documentation an agent reads, copied in at build (packages/mcp/scripts/copy-docs.mjs)
+/packages/mcp/docs/

apps/vscode-extension/e2e/suite/index.cjs

@@ -73,7 +73,7 @@ async function run () {
 
   // The MCP server the extension registers with the editor's agent mode starts, and offers OpenCraw's tools.
   const tools = await mcpToolNames(api.mcpServerScript())
-  for (const name of ['probe', 'validate', 'run', 'list_recipes']) assert.ok(tools.includes(name), `the MCP server offers ${name} (offers: ${tools.join(', ')})`)
+  for (const name of ['probe', 'validate', 'run', 'list_recipes', 'docs']) assert.ok(tools.includes(name), `the MCP server offers ${name} (offers: ${tools.join(', ')})`)
 
   // The recipes view lists the workspace's recipes by the output they fill: an input whose output is missing is kept, under its name.
   const groups = await api.recipeGroups()

packages/mcp/README.md

@@ -11,7 +11,7 @@ directly: probe a page, validate recipes, run a crawl, list what's already autho
 For a shared, remote endpoint that authors recipes where they will run (and publishes them to a crawl host), see
 [`@opencraw/azure-durable`](https://github.com/Open-Craw/OpenCraw/blob/main/packages/azure-durable/README.md#write-recipes-over-mcp)'s `/mcp`.
 
-This does **not** auto-author recipes from a sentence. The five tools are, `list_recipes` aside, the same
+This does **not** auto-author recipes from a sentence. The crawl tools are, `list_recipes` and `docs` aside, the same
 primitives the `opencraw` cli gives a terminal; the calling agent still writes the JSON recipes, using `probe` and
 `validate` to iterate, the same way this project's own example recipes were built by hand. An agent that
 can write files passes their `paths`; one that can't (a chat-only host) passes the `recipes` inline.
@@ -116,6 +116,19 @@ call, for checking what already exists before writing a new recipe.
 
 Returns `{ outputs: [{ path, id }], inputs: [{ path, id, output, mode }], others: string[] }`.
 
+### `docs`
+
+OpenCraw's own documentation, for an agent that has to write recipes right: the authoring guide (every step, selector, transform and policy), the
+how-it-works chapters, the examples and the Studio manual. They ship inside the package.
+
+| Input | Meaning |
+|---|---|
+| `topic?` | A document id (`recipes/authoring`, `how-it-works/06-documents`, `studio/manual`…). Leave out to list every document, with its title and size. |
+| `section?` | A heading (or part of one) of that document: only that section comes back, subsections included. A long document answers with its outline until a section is asked for. |
+
+Returns `{ topics, hint }` without a topic, `{ topic, text }` (or `{ topic, outline, note }` for a long document) with one, and `{ topic, section, text }`
+with a section.
+
 ### `diff`
 
 Compares two runs' JSON Lines files by record key: what was added, what was removed, and which fields changed,

packages/mcp/e2e/mcp.e2e.test.ts

@@ -40,9 +40,17 @@ function oneLine (path: string): string {
 }
 
 describe('opencraw mcp server', () => {
-  it('lists the four crawl-primitive tools', async () => {
+  it('lists the crawl-primitive tools and the documentation', async () => {
     const { tools } = await client.listTools()
-    expect(tools.map(tool => tool.name).sort((a, b) => a.localeCompare(b))).toEqual(['diff', 'list_recipes', 'probe', 'run', 'validate'])
+    expect(tools.map(tool => tool.name).sort((a, b) => a.localeCompare(b))).toEqual(['diff', 'docs', 'list_recipes', 'probe', 'run', 'validate'])
+  })
+
+  it('docs: lists the documents, and gives a section of the authoring guide', async () => {
+    const listed = textOf(await client.callTool({ name: 'docs', arguments: {} })) as { topics: { id: string }[] }
+    expect(listed.topics.map(topic => topic.id)).toEqual(expect.arrayContaining(['recipes/authoring', 'how-it-works/06-documents', 'studio/manual']))
+
+    const section = textOf(await client.callTool({ name: 'docs', arguments: { topic: 'recipes/authoring', section: 'hooks' } })) as { text?: string }
+    expect(section.text).toMatch(/hook/i)
   })
 
   it('validate: accepts the reference recipes and reports the binding', async () => {

packages/mcp/package.json

@@ -19,6 +19,7 @@
   "files": [
     "dist",
     "bin",
+    "docs",
     "!**/*.tsbuildinfo",
     "!**/*.d.ts.map",
     "!**/*.js.map"
@@ -78,7 +79,8 @@
               "@opencraw/cli"
             ],
             "target": "e2e"
-          }
+          },
+          "copy-docs"
         ],
         "inputs": [
           "default",
@@ -89,6 +91,29 @@
           "command": "tsc -p tsconfig.e2e.json && jest --config jest.e2e.config.cts",
           "cwd": "packages/mcp"
         }
+      },
+      "copy-docs": {
+        "executor": "nx:run-commands",
+        "cache": true,
+        "inputs": [
+          "{workspaceRoot}/docs/recipes/*.md",
+          "{workspaceRoot}/docs/how-it-works/*.md",
+          "{workspaceRoot}/docs/studio/*.md",
+          "{projectRoot}/scripts/copy-docs.mjs"
+        ],
+        "outputs": [
+          "{projectRoot}/docs"
+        ],
+        "options": {
+          "command": "node scripts/copy-docs.mjs",
+          "cwd": "packages/mcp"
+        }
+      },
+      "build": {
+        "dependsOn": [
+          "^build",
+          "copy-docs"
+        ]
       }
     }
   }

packages/mcp/scripts/copy-docs.mjs

@@ -0,0 +1,16 @@
+// Copies the documentation an agent reads (the `docs` tool) into the package, so it ships with it: `docs/recipes`, `docs/how-it-works` and `docs/studio`,
+// the Markdown files directly in each (their pictures and capture scripts stay behind).
+import { cpSync, mkdirSync, readdirSync, rmSync } from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const here = dirname(fileURLToPath(import.meta.url))
+const source = resolve(here, '..', '..', '..', 'docs')
+const target = resolve(here, '..', 'docs')
+
+rmSync(target, { recursive: true, force: true })
+for (const folder of ['recipes', 'how-it-works', 'studio']) {
+  mkdirSync(join(target, folder), { recursive: true })
+  const documents = readdirSync(join(source, folder)).filter(file => file.toLowerCase().endsWith('.md'))
+  for (const name of documents) cpSync(join(source, folder, name), join(target, folder, name))
+}

packages/mcp/src/docs-tool/docs-folder.repository.ts

@@ -0,0 +1,55 @@
+import { readdir, readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { titleOf, topicIdOf } from './docs-topics.algorithm'
+
+/** One document the server can show. */
+export interface DocTopic {
+  id:    string
+  title: string
+  /** Its size in characters, so an agent knows to ask for a section of a long one. */
+  size:  number
+}
+
+/** The folders under the docs folder whose Markdown files are documentation for a person (or an agent) using OpenCraw. */
+const FOLDERS = ['recipes', 'how-it-works', 'studio']
+
+/**
+ * The documents under a docs folder: the Markdown files directly inside `recipes`, `how-it-works` and `studio` (their pictures and capture scripts are
+ * not documentation to read).
+ *
+ * @param root - The docs folder.
+ * @returns The topics, ordered by id.
+ */
+export async function listTopics (root: string): Promise<DocTopic[]> {
+  const topics: DocTopic[] = []
+  for (const folder of FOLDERS) {
+    let names: string[]
+    try {
+      names = await readdir(join(root, folder))
+    } catch {
+      continue
+    }
+    const documents = names.filter(file => file.toLowerCase().endsWith('.md'))
+    for (const name of documents) {
+      const id = topicIdOf(`${folder}/${name}`)
+      const text = await readFile(join(root, folder, name), 'utf8')
+      topics.push({ id, title: titleOf(text, id), size: text.length })
+    }
+  }
+
+  return topics.sort((a, b) => a.id.localeCompare(b.id))
+}
+
+/**
+ * One document's text.
+ *
+ * @param root - The docs folder.
+ * @param id - The topic id from `listTopics` (never a path: nothing outside the listed folders is read).
+ * @returns The Markdown, or `undefined` when there is no such topic.
+ */
+export async function readTopic (root: string, id: string): Promise<string | undefined> {
+  const known = await listTopics(root)
+  if (known.every(topic => topic.id !== id)) return undefined
+
+  return await readFile(join(root, `${id}.md`), 'utf8')
+}

packages/mcp/src/docs-tool/docs-root.client.ts

@@ -0,0 +1,16 @@
+import { existsSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+/**
+ * Where the documentation is: a `docs` folder beside `dist` in the installed package (copied there at build), or the repository's own `docs` when the server runs
+ * from a checkout.
+ *
+ * @returns The docs folder.
+ */
+export function docsRoot (): string {
+  const here = dirname(fileURLToPath(import.meta.url))
+  const candidates = [join(here, '..', 'docs'), join(here, '..', '..', '..', 'docs')]
+
+  return candidates.find(candidate => existsSync(join(candidate, 'recipes'))) ?? candidates[0]
+}

packages/mcp/src/docs-tool/docs-tool.handler.test.ts

@@ -0,0 +1,111 @@
+import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'
+import { docsTool } from './docs-tool.handler'
+import { outlineOf, sectionOf, titleOf, topicIdOf } from './docs-topics.algorithm'
+
+const AUTHORING = `# Authoring guide
+
+Intro text.
+
+## 1. Recipes
+
+About recipes.
+
+### 1.1 Input
+
+An input recipe.
+
+\`\`\`sh
+# not a heading
+\`\`\`
+
+## 2. Selectors
+
+About selectors.
+`
+
+async function docsFolder (long = false): Promise<string> {
+  const root = await mkdtemp(join(tmpdir(), 'mcp-docs-'))
+  await mkdir(join(root, 'recipes'))
+  await mkdir(join(root, 'studio'))
+  await mkdir(join(root, 'capture'))
+  await writeFile(join(root, 'recipes', 'authoring.md'), long ? `${AUTHORING}\n${'x'.repeat(40_000)}` : AUTHORING)
+  await writeFile(join(root, 'studio', 'manual.md'), '# Studio manual\n\nClick.\n')
+  await writeFile(join(root, 'capture', 'script.md'), '# not documentation\n')
+
+  return root
+}
+
+function body (result: CallToolResult): Record<string, unknown> {
+  const first = result.content[0] as { text: string }
+
+  return JSON.parse(first.text) as Record<string, unknown>
+}
+
+describe('docs-topics', () => {
+  it('names a topic by its path under the docs folder, and its title by its first heading', () => {
+    expect(topicIdOf(String.raw`recipes\authoring.md`)).toBe('recipes/authoring')
+    expect(titleOf(AUTHORING, 'x')).toBe('Authoring guide')
+    expect(titleOf('no heading', 'fallback')).toBe('fallback')
+  })
+
+  it('lists the headings and ignores a # inside a code fence', () => {
+    expect(outlineOf(AUTHORING).map(heading => `${String(heading.level)} ${heading.text}`)).toEqual(['1 Authoring guide', '2 1. Recipes', '3 1.1 Input', '2 2. Selectors'])
+  })
+
+  it('takes a section with its subsections up to the next heading of its level, by exact or partial heading', () => {
+    expect(sectionOf(AUTHORING, '1. Recipes')).toContain('1.1 Input')
+    expect(sectionOf(AUTHORING, '1. Recipes')).not.toContain('About selectors')
+    expect(sectionOf(AUTHORING, 'selectors')).toContain('About selectors')
+    expect(sectionOf(AUTHORING, 'nothing like it')).toBeUndefined()
+  })
+})
+
+describe('docsTool', () => {
+  it('lists the documents of the docs folders, not what sits in other folders', async () => {
+    const result = body(await docsTool({}, await docsFolder()))
+
+    expect((result.topics as { id: string, title: string }[]).map(topic => [topic.id, topic.title])).toEqual([['recipes/authoring', 'Authoring guide'], ['studio/manual', 'Studio manual']])
+  })
+
+  it('gives a short document whole, and a section of it when asked', async () => {
+    const root = await docsFolder()
+
+    expect(body(await docsTool({ topic: 'studio/manual' }, root)).text).toContain('Click.')
+    expect(body(await docsTool({ topic: 'recipes/authoring', section: 'selectors' }, root)).text).toContain('## 2. Selectors')
+  })
+
+  it('answers a long document with its outline until a section is asked for', async () => {
+    const result = body(await docsTool({ topic: 'recipes/authoring' }, await docsFolder(true)))
+
+    expect(result.text).toBeUndefined()
+    expect(result.outline).toContain('## 2. Selectors')
+    expect(String(result.note)).toContain('ask for a section')
+  })
+
+  it('says what is wrong for an unknown topic or heading, with what to try', async () => {
+    const root = await docsFolder()
+    const unknown = await docsTool({ topic: 'authoring' }, root)
+    const wrongHeading = await docsTool({ topic: 'recipes/authoring', section: 'zzz' }, root)
+
+    expect(unknown.isError).toBe(true)
+    expect((unknown.content[0] as { text: string }).text).toContain('recipes/authoring')
+    expect(wrongHeading.isError).toBe(true)
+    expect(body(wrongHeading).outline).toContain('## 1. Recipes')
+  })
+
+  it('never reads outside the listed folders: a path is not a topic', async () => {
+    const result = await docsTool({ topic: '../../etc/passwd' }, await docsFolder())
+
+    expect(result.isError).toBe(true)
+  })
+
+  it('says so when no documentation is installed', async () => {
+    const empty = await mkdtemp(join(tmpdir(), 'mcp-docs-empty-'))
+    const result = await docsTool({}, empty)
+
+    expect(result.isError).toBe(true)
+  })
+})

packages/mcp/src/docs-tool/docs-tool.handler.ts

@@ -0,0 +1,48 @@
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'
+import { z } from 'zod'
+import { listTopics, readTopic } from './docs-folder.repository'
+import { outlineOf, sectionOf } from './docs-topics.algorithm'
+
+/** Input schema for the `docs` tool. */
+export const docsInputShape = {
+  topic:   z.string().optional().describe('A document id from the list this tool returns without a topic (`recipes/authoring`, `how-it-works/06-documents`, `studio/manual`…). Leave out to list every document.'),
+  section: z.string().optional().describe('A heading (or part of one) of that document: only that section comes back, subsections included. A long document answers with its outline until you ask for a section.'),
+}
+
+/** A document longer than this many characters comes back as its outline, and the section asked for. */
+const WHOLE_DOCUMENT_LIMIT = 30_000
+
+function text (value: unknown, isError = false): CallToolResult {
+  return { content: [{ type: 'text', text: typeof value === 'string' ? value : JSON.stringify(value) }], ...(isError && { isError }), ...(typeof value === 'object' && value !== null && { structuredContent: value as Record<string, unknown> }) }
+}
+
+/**
+ * The `docs` tool: OpenCraw's own documentation, for an agent that has to write recipes right: the authoring guide (every step, selector, transform
+ * and policy), how it all works, the examples and the Studio manual. Without a topic it lists what there is; with one it gives the document, or just
+ * the section asked for, so a long guide costs only what is read.
+ *
+ * @param args - The tool's parsed input.
+ * @param root - The docs folder (the packaged copy, or the repository's).
+ * @returns The MCP tool result.
+ */
+export async function docsTool (args: { topic?: string, section?: string }, root: string): Promise<CallToolResult> {
+  const topics = await listTopics(root)
+  if (topics.length === 0) return text('The documentation is not installed with this server.', true)
+  if (args.topic === undefined) return text({ topics, hint: 'Ask for one with { topic }, and for part of a long one with { topic, section }.' })
+  const document = await readTopic(root, args.topic)
+  if (document === undefined) {
+    const close = topics.filter(topic => topic.id.includes(args.topic?.toLowerCase() ?? '') || args.topic?.includes(topic.id) === true).map(topic => topic.id)
+
+    return text(`There is no document "${args.topic}". ${close.length > 0 ? `Did you mean: ${close.join(', ')}?` : `The documents are: ${topics.map(topic => topic.id).join(', ')}.`}`, true)
+  }
+  if (args.section !== undefined) {
+    const section = sectionOf(document, args.section)
+
+    return section === undefined
+      ? text({ topic: args.topic, error: `no heading contains "${args.section}"`, outline: outlineOf(document).map(heading => `${'#'.repeat(heading.level)} ${heading.text}`) }, true)
+      : text({ topic: args.topic, section: args.section, text: section })
+  }
+  if (document.length <= WHOLE_DOCUMENT_LIMIT) return text({ topic: args.topic, text: document })
+
+  return text({ topic: args.topic, outline: outlineOf(document).map(heading => `${'#'.repeat(heading.level)} ${heading.text}`), note: `This document is ${String(document.length)} characters: ask for a section with { topic, section } using a heading from the outline.` })
+}