diff --git a/.github/workflows/vscode-e2e.yml b/.github/workflows/vscode-e2e.yml index 20cb96d0..14aae62d 100644 --- a/.github/workflows/vscode-e2e.yml +++ b/.github/workflows/vscode-e2e.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/cache@v4 with: path: apps/vscode-extension/.vscode-test - key: vscode-test-${{ runner.os }}-1.90.0 + key: vscode-test-${{ runner.os }}-1.127.0 - name: The extension in a real editor, on a virtual display run: xvfb-run -a npm run vscode:e2e env: diff --git a/apps/vscode-extension/README.md b/apps/vscode-extension/README.md index 35e5f2d2..ef39fa2c 100644 --- a/apps/vscode-extension/README.md +++ b/apps/vscode-extension/README.md @@ -15,6 +15,21 @@ Issue #228. The server side it relies on is `choose-folder` (#234, the Browse bu recipe from it, the same as dropping the file on Studio. A file inside the workspace folder is read in place; one from outside is copied into the workspace folder. +## The OpenCraw sidebar + +- **Recipes** lists the recipes in your workspace (`*.ocw-in.json`, `*.ocw-out.json`, and the `*.input.json` / `*.output.json` the examples use) + **grouped by output id**: each output recipe is a row, the input recipes that fill it are under it. + - **Run** on an input recipe runs that one; on an output recipe it runs every input that feeds it. The crawl runs in a process of its own with + a notification you can cancel, and its messages go to the **OpenCraw** output. + - When the crawl is over you are asked where to save the result with your operating system's own save dialog. **The file's extension is the + format**: `.xlsx` (Excel), `.csv`, `.html` (a page that looks like Studio), `.pdf`, `.json`, `.jsonl` or `.yml`/`.yaml`. A name that is none + of these is explained and the dialog opens again. + - **Open in Studio** opens the recipe to change it. +- **Learn** links the Studio manual, the tutorial, the recipe guides and where to ask. +- **OpenCraw: Get started** opens a walkthrough (it also opens once, the first time the extension runs). + +Needs VS Code 1.127 (June 2026) or newer. + The server starts on first use, in a process of its own, and stops with the window. In a remote window (SSH, WSL, Codespaces, dev containers) it runs where the workspace is and its port is forwarded to the browser on your machine. @@ -33,6 +48,11 @@ editor (extension host) Studio process browser - `src/studio-host`: forks the Studio process and asks it for links over IPC. The process is the editor's own executable run as plain Node (`ELECTRON_RUN_AS_NODE`), so the extension ships no Node. - `src/open-on-opencraw`: the command's outcome, the only place that touches the editor API. +- `src/recipe-explorer`: finds the recipe files, reads each one's `kind`/`id`/`output` (the extension does not carry the engine), groups them by + output id, and is the recipes view. +- `src/recipe-run`: the Run action: the plan (which files, `--only`), the process that runs the bundled `opencraw` command line, the save dialog + and the format from the extension, and `opencraw export` to write the records in it. +- `src/learn-studio`: the Learn view and the Get started walkthrough (`media/walkthrough`). - `src/extension.ts`: the entry point the editor requires. **Recipe ▾ → Open folder…** in Studio shows the editor's own folder dialog (`vscode.window.showOpenDialog`): the @@ -95,7 +115,7 @@ Open VSX is skipped when `OVSX_PAT` is not set. ## End-to-end test -`npm run vscode:e2e` stages the extension as the editor loads it, starts a real VS Code (1.90.0, the oldest the +`npm run vscode:e2e` stages the extension as the editor loads it, starts a real VS Code (1.127.0, the oldest the extension supports; downloaded into `.vscode-test` on the first run) and runs `e2e/suite/index.cjs` in its extension host: the commands exist, "Open on OpenCraw" resolves a link to a real Studio process, the page is served, `open-path` answers, and the `opencraw.hooksFile` setting reaches the server. The browser the link is diff --git a/apps/vscode-extension/e2e/run-editor-test.mjs b/apps/vscode-extension/e2e/run-editor-test.mjs index f47e189d..e7e84df5 100644 --- a/apps/vscode-extension/e2e/run-editor-test.mjs +++ b/apps/vscode-extension/e2e/run-editor-test.mjs @@ -9,12 +9,17 @@ import { runTests } from '@vscode/test-electron' const appDir = resolve(dirname(fileURLToPath(import.meta.url)), '..') /** The oldest editor the extension claims to support (`engines.vscode`). */ -const VERSION = '1.90.0' +const VERSION = '1.127.0' const workspace = mkdtempSync(join(tmpdir(), 'opencraw-editor-workspace-')) const profile = mkdtempSync(join(tmpdir(), 'opencraw-editor-profile-')) mkdirSync(join(workspace, 'docs')) writeFileSync(join(workspace, 'docs', 'report.csv'), 'name,price\nbook,10\n') +// Recipes for the recipes view: two inputs that fill one output, and an input whose output is not in the workspace. +writeFileSync(join(workspace, 'cars.ocw-out.json'), JSON.stringify({ kind: 'output', id: 'cars', version: 1, fields: { model: { type: 'string' } } })) +writeFileSync(join(workspace, 'kia.ocw-in.json'), JSON.stringify({ kind: 'input', id: 'kia', output: 'cars', mode: 'api', start: [], steps: [] })) +writeFileSync(join(workspace, 'byd.ocw-in.json'), JSON.stringify({ kind: 'input', id: 'byd', output: 'cars', mode: 'api', start: [], steps: [] })) +writeFileSync(join(workspace, 'lost.ocw-in.json'), JSON.stringify({ kind: 'input', id: 'lost', output: 'elsewhere', mode: 'api', start: [], steps: [] })) try { await runTests({ diff --git a/apps/vscode-extension/e2e/suite/index.cjs b/apps/vscode-extension/e2e/suite/index.cjs index f6811de3..46984df2 100644 --- a/apps/vscode-extension/e2e/suite/index.cjs +++ b/apps/vscode-extension/e2e/suite/index.cjs @@ -41,6 +41,11 @@ async function run () { const registered = await vscode.commands.getCommands(true) assert.ok(registered.includes('opencraw.openStudio'), 'Open Studio is registered') assert.ok(registered.includes('opencraw.openOn'), 'Open on OpenCraw is registered') + for (const name of ['runRecipe', 'openRecipeInStudio', 'refreshRecipes', 'showWalkthrough', 'openLink']) assert.ok(registered.includes(`opencraw.${name}`), `${name} is registered`) + + // The recipes view lists the workspace's recipes by the output they fill: an input whose output is missing is kept, under its name. + const groups = await api.recipeGroups() + assert.deepEqual(groups.map(group => [group.outputId, group.outputFile !== undefined, group.inputs.map(input => input.id)]), [['cars', true, ['byd', 'kia']], ['elsewhere', false, ['lost']]]) const folder = vscode.workspace.workspaceFolders[0].uri const report = vscode.Uri.joinPath(folder, 'docs', 'report.csv') diff --git a/apps/vscode-extension/media/walkthrough/collect.md b/apps/vscode-extension/media/walkthrough/collect.md new file mode 100644 index 00000000..84975588 --- /dev/null +++ b/apps/vscode-extension/media/walkthrough/collect.md @@ -0,0 +1,13 @@ +# Click what you want + +Click the value you want to collect. Studio shows the ways to find it again on the next visit, with how many things each one matches: + +- the suggested one is the most reliable; +- an id that changes on every visit is never used; +- you can type your own. + +Click a **second, similar** value (the next product's name, say) and Studio collects the **whole list**. + +When a way to find it matches several things, you choose: only the first, or all of them as a list. + +Then press **Add to recipe**. diff --git a/apps/vscode-extension/media/walkthrough/load.md b/apps/vscode-extension/media/walkthrough/load.md new file mode 100644 index 00000000..53b1f9ec --- /dev/null +++ b/apps/vscode-extension/media/walkthrough/load.md @@ -0,0 +1,10 @@ +# Load what you want to read + +In Studio press **Load** and choose: + +- **Web page**: paste an address. Cookie banners are closed for you. +- **Document**: a PDF, a spreadsheet (xlsx, csv), a deck, a Word file, JSON, YAML, XML or Markdown. + +Whatever you load appears on the left, the way it looks: a page, a PDF page, a grid or a tree. The steps of the recipe are on the right. + +Big files are fine: a long spreadsheet or JSON shows only the part you are looking at. diff --git a/apps/vscode-extension/media/walkthrough/open-studio.md b/apps/vscode-extension/media/walkthrough/open-studio.md new file mode 100644 index 00000000..bfad2a8f --- /dev/null +++ b/apps/vscode-extension/media/walkthrough/open-studio.md @@ -0,0 +1,7 @@ +# Open Studio + +Studio is where you build recipes by clicking. It starts on your computer, for this workspace, and opens in your browser: nothing is sent anywhere. + +Use **OpenCraw: Open Studio** from the command palette, or right-click any file or folder and choose **Open on OpenCraw**. A PDF, a spreadsheet, a Word file, a JSON or Markdown file starts a recipe straight away. + +The first time can take a few seconds while the server starts. diff --git a/apps/vscode-extension/media/walkthrough/recipes.md b/apps/vscode-extension/media/walkthrough/recipes.md new file mode 100644 index 00000000..8c36ca05 --- /dev/null +++ b/apps/vscode-extension/media/walkthrough/recipes.md @@ -0,0 +1,8 @@ +# Run it from the editor + +The **OpenCraw** view in the sidebar lists your recipes, grouped by the output they fill. + +- **Run** on an input recipe runs just that one. +- **Run** on an output recipe runs every input that feeds it. +- When the crawl is over you choose where to save the result, and its format is the file's extension: Excel (`.xlsx`), CSV, a web page (`.html`), PDF, JSON, JSON Lines or YAML. +- **Open in Studio** opens the recipe to change it. diff --git a/apps/vscode-extension/media/walkthrough/run.md b/apps/vscode-extension/media/walkthrough/run.md new file mode 100644 index 00000000..c7f9ccc0 --- /dev/null +++ b/apps/vscode-extension/media/walkthrough/run.md @@ -0,0 +1,10 @@ +# Run a sample + +Press **Run sample**. The records appear in a table at the bottom, and the **Trace** tab tells you what the crawl did, step by step. + +If a value is missing or wrong, **Why?** explains which step produced it and what it saw. + +When it looks right, the recipe is already saved in your folder as two files: + +- an **input recipe** (`.ocw-in.json`): where to read, and what to take; +- an **output recipe** (`.ocw-out.json`): the shape of the result. Different inputs can fill the same output, so different sources give you the same kind of data. diff --git a/apps/vscode-extension/package.json b/apps/vscode-extension/package.json index 11db59b3..cea7c384 100644 --- a/apps/vscode-extension/package.json +++ b/apps/vscode-extension/package.json @@ -1,15 +1,15 @@ { "name": "@opencraw/vscode", - "version": "0.1.6", + "version": "0.2.0", "private": true, - "description": "OpenCraw in the editor: starts the Studio server and opens a folder, recipe or document in the browser.", + "description": "OpenCraw in the editor: open folders, recipes and documents in Studio, run your recipes from a view and save the result in the format you choose, and learn how it all works.", "type": "module", "main": "dist/extension.cjs", "displayName": "OpenCraw", "publisher": "opencraw", "icon": "media/icon.png", "engines": { - "vscode": "^1.90.0" + "vscode": "^1.127.0" }, "categories": [ "Other" @@ -28,15 +28,23 @@ "views": { "opencraw": [ { - "id": "opencraw.start", - "name": "OpenCraw" + "id": "opencraw.recipes", + "name": "Recipes", + "icon": "media/opencraw-activity-bar.svg", + "contextualTitle": "OpenCraw recipes" + }, + { + "id": "opencraw.learn", + "name": "Learn", + "icon": "media/opencraw-activity-bar.svg", + "contextualTitle": "Learn OpenCraw" } ] }, "viewsWelcome": [ { - "view": "opencraw.start", - "contents": "Run recipes in your browser: Studio starts a local server and opens the page.\n[Open Studio](command:opencraw.openStudio)\nTo open a folder, recipe or document, right-click it in the Explorer and choose **Open on OpenCraw**." + "view": "opencraw.recipes", + "contents": "No recipes in this folder yet.\nStudio builds them for you: load a web page or a document, click what you want, and it writes the recipe.\n[Open Studio](command:opencraw.openStudio)\nRecipes are the files ending in `.ocw-in.json` (what to read) and `.ocw-out.json` (the shape of the result). They are grouped here by the output they fill." } ], "jsonValidation": [ @@ -74,6 +82,35 @@ "command": "opencraw.openOn", "title": "Open on OpenCraw", "category": "OpenCraw" + }, + { + "command": "opencraw.runRecipe", + "title": "Run", + "category": "OpenCraw", + "icon": "$(play)" + }, + { + "command": "opencraw.openRecipeInStudio", + "title": "Open in Studio", + "category": "OpenCraw", + "icon": "$(browser)" + }, + { + "command": "opencraw.refreshRecipes", + "title": "Refresh the recipes", + "category": "OpenCraw", + "icon": "$(refresh)" + }, + { + "command": "opencraw.showWalkthrough", + "title": "Get started", + "category": "OpenCraw", + "icon": "$(mortar-board)" + }, + { + "command": "opencraw.openLink", + "title": "Open a link", + "category": "OpenCraw" } ], "menus": { @@ -102,9 +139,115 @@ { "command": "opencraw.openOn", "when": "resourceScheme == file" + }, + { + "command": "opencraw.runRecipe", + "when": "false" + }, + { + "command": "opencraw.openRecipeInStudio", + "when": "false" + }, + { + "command": "opencraw.openLink", + "when": "false" + } + ], + "view/title": [ + { + "command": "opencraw.openStudio", + "when": "view == opencraw.recipes", + "group": "navigation@1" + }, + { + "command": "opencraw.refreshRecipes", + "when": "view == opencraw.recipes", + "group": "navigation@2" + }, + { + "command": "opencraw.showWalkthrough", + "when": "view == opencraw.learn", + "group": "navigation@1" + } + ], + "view/item/context": [ + { + "command": "opencraw.runRecipe", + "when": "view == opencraw.recipes && viewItem =~ /^opencraw\\.(output\\.runnable|input)$/", + "group": "inline@1" + }, + { + "command": "opencraw.openRecipeInStudio", + "when": "view == opencraw.recipes && viewItem =~ /^opencraw\\.(output|input)/ && viewItem != opencraw.output.missing", + "group": "inline@2" + }, + { + "command": "opencraw.runRecipe", + "when": "view == opencraw.recipes && viewItem =~ /^opencraw\\.(output\\.runnable|input)$/", + "group": "navigation@1" + }, + { + "command": "opencraw.openRecipeInStudio", + "when": "view == opencraw.recipes && viewItem =~ /^opencraw\\.(output|input)/ && viewItem != opencraw.output.missing", + "group": "navigation@2" } ] - } + }, + "walkthroughs": [ + { + "id": "opencraw.getStarted", + "title": "Get started with OpenCraw", + "description": "Collect data from a web page, a PDF, a spreadsheet or a file, by clicking: no code.", + "steps": [ + { + "id": "open", + "title": "Open Studio", + "description": "Studio runs on your computer and opens in your browser, on this workspace.\n[Open Studio](command:opencraw.openStudio)", + "media": { + "markdown": "media/walkthrough/open-studio.md" + }, + "completionEvents": [ + "onCommand:opencraw.openStudio" + ] + }, + { + "id": "load", + "title": "Load what you want to read", + "description": "A web address, a PDF, a spreadsheet, a Word file, JSON: Load puts it on the left of Studio.", + "media": { + "markdown": "media/walkthrough/load.md" + } + }, + { + "id": "collect", + "title": "Click what you want", + "description": "Click a value to collect it. Click a second, similar one to collect the whole list. Pick the way to find it that will keep working.", + "media": { + "markdown": "media/walkthrough/collect.md" + } + }, + { + "id": "run", + "title": "Run a sample", + "description": "Run sample shows the records in a table. Fix what looks wrong, then save.", + "media": { + "markdown": "media/walkthrough/run.md" + } + }, + { + "id": "recipes", + "title": "Run it from the editor", + "description": "Your recipes are listed here, grouped by the output they fill. Run one, or run them all, and choose where to save the result and in what format.\n[Show the recipes](command:opencraw.recipes.focus)", + "media": { + "markdown": "media/walkthrough/recipes.md" + }, + "completionEvents": [ + "onCommand:opencraw.runRecipe" + ] + } + ] + } + ] }, "nx": { "targets": { @@ -192,7 +335,7 @@ "@opencraw/studio": "^0.0.1" }, "devDependencies": { - "@types/vscode": "1.90.0", + "@types/vscode": "1.125.0", "@vscode/test-electron": "^3.1.0" } } diff --git a/apps/vscode-extension/src/extension.ts b/apps/vscode-extension/src/extension.ts index d7e009e1..77c7b1c5 100644 --- a/apps/vscode-extension/src/extension.ts +++ b/apps/vscode-extension/src/extension.ts @@ -1,26 +1,73 @@ import * as vscode from 'vscode' +import { LearnTree } from './learn-studio' import { createStudioRunner } from './open-on-opencraw' +import { RecipeTree, createRecipeIndex, groupByOutput } from './recipe-explorer' +import type { RecipeGroup, RecipeNode } from './recipe-explorer' +import { createRecipeRunner } from './recipe-run' /** What the extension hands to code that activates it: other extensions, and the editor end-to-end test. */ export interface OpenCrawApi { /** Fires with each link about to be opened in the browser. */ onDidResolveLink: vscode.Event + /** The recipes the recipes view lists now, grouped by the output they fill. */ + recipeGroups: () => Promise +} + +/** The extension's id in the editor, for the walkthrough's address. */ +const WALKTHROUGH = 'opencraw.opencraw#opencraw.getStarted' + +/** The hooks module the person named in their user settings; `undefined` when none (a relative or empty setting is left to the Studio side to report). */ +function hooksSetting (): string | undefined { + const named = vscode.workspace.getConfiguration('opencraw').get('hooksFile', '').trim() + + return named === '' ? undefined : named +} + +/** The file a recipes-view row opens in Studio: an input's own file, an output group's output recipe. */ +function fileOf (node: RecipeNode): string | undefined { + return node.type === 'input' ? node.header.file : node.group.outputFile } /** The extension's entry point, as the editor requires it. */ export function activate (context: vscode.ExtensionContext): OpenCrawApi { const runner = createStudioRunner(context.extensionPath) + const output = vscode.window.createOutputChannel('OpenCraw') + const index = createRecipeIndex() + const recipes = new RecipeTree(index) + const recipeRunner = createRecipeRunner({ extensionPath: context.extensionPath, output, hooksFile: hooksSetting }) context.subscriptions.push( + output, + index, + recipes, + vscode.window.createTreeView('opencraw.recipes', { treeDataProvider: recipes, showCollapseAll: true }), + vscode.window.createTreeView('opencraw.learn', { treeDataProvider: new LearnTree() }), vscode.commands.registerCommand('opencraw.openStudio', async () => { await runner.open() }), vscode.commands.registerCommand('opencraw.openOn', async (uri?: vscode.Uri) => { await runner.open(uri) }), + vscode.commands.registerCommand('opencraw.runRecipe', async (node?: RecipeNode) => { if (node !== undefined) await recipeRunner.run(node) }), + vscode.commands.registerCommand('opencraw.openRecipeInStudio', async (node?: RecipeNode) => { + const file = node === undefined ? undefined : fileOf(node) + if (file !== undefined) await runner.open(vscode.Uri.file(file)) + }), + vscode.commands.registerCommand('opencraw.refreshRecipes', () => { recipes.refresh() }), + vscode.commands.registerCommand('opencraw.showWalkthrough', async () => { await vscode.commands.executeCommand('workbench.action.openWalkthrough', WALKTHROUGH, false) }), + vscode.commands.registerCommand('opencraw.openLink', async (url?: string) => { if (url !== undefined) await vscode.env.openExternal(vscode.Uri.parse(url)) }), // The hooks file is read when the server starts: a changed setting takes effect from the next open. vscode.workspace.onDidChangeConfiguration(change => { if (change.affectsConfiguration('opencraw.hooksFile')) runner.restart() }), { dispose: () => { runner.dispose() } }, ) + void showWalkthroughOnFirstRun(context) + + return { onDidResolveLink: runner.onDidResolveLink, recipeGroups: async () => groupByOutput(await index.headers()) } +} - return { onDidResolveLink: runner.onDidResolveLink } +/** The first time the extension runs on a machine, the Get Started page opens once: a person who has never seen it learns what it does. */ +async function showWalkthroughOnFirstRun (context: vscode.ExtensionContext): Promise { + const KEY = 'opencraw.walkthroughShown' + if (context.globalState.get(KEY) === true) return + await context.globalState.update(KEY, true) + if (context.extensionMode === vscode.ExtensionMode.Production) await vscode.commands.executeCommand('opencraw.showWalkthrough') } export function deactivate (): void {} diff --git a/apps/vscode-extension/src/learn-studio/index.ts b/apps/vscode-extension/src/learn-studio/index.ts new file mode 100644 index 00000000..59824d0a --- /dev/null +++ b/apps/vscode-extension/src/learn-studio/index.ts @@ -0,0 +1,3 @@ +export { LearnTree } from './learn-tree.client' +export { LEARN_LINKS } from './learn-links.config' +export type { LearnLink } from './learn-links.config' diff --git a/apps/vscode-extension/src/learn-studio/learn-links.config.test.ts b/apps/vscode-extension/src/learn-studio/learn-links.config.test.ts new file mode 100644 index 00000000..0f552cbc --- /dev/null +++ b/apps/vscode-extension/src/learn-studio/learn-links.config.test.ts @@ -0,0 +1,13 @@ +import { LEARN_LINKS } from './learn-links.config' + +describe('LEARN_LINKS', () => { + it('are all web addresses, each once, each with a label and a description', () => { + expect(LEARN_LINKS.length).toBeGreaterThan(5) + expect(new Set(LEARN_LINKS.map(link => link.url)).size).toBe(LEARN_LINKS.length) + for (const link of LEARN_LINKS) { + expect(link.url).toMatch(/^https:\/\/github\.com\/Open-Craw\/OpenCraw\//) + expect(link.label).not.toBe('') + expect(link.description).not.toBe('') + } + }) +}) diff --git a/apps/vscode-extension/src/learn-studio/learn-links.config.ts b/apps/vscode-extension/src/learn-studio/learn-links.config.ts new file mode 100644 index 00000000..8423d018 --- /dev/null +++ b/apps/vscode-extension/src/learn-studio/learn-links.config.ts @@ -0,0 +1,22 @@ +/** One place to read about OpenCraw, shown in the Learn view. */ +export interface LearnLink { + label: string + description: string + url: string + /** A built-in editor icon name (a codicon). */ + icon: string +} + +const DOCS = 'https://github.com/Open-Craw/OpenCraw/blob/main/docs' + +/** What a newcomer should read, in the order they will want it: Studio first, then recipes, then how it all works, then where to ask. */ +export const LEARN_LINKS: readonly LearnLink[] = [ + { label: 'Studio manual', description: 'Every button, what it does and when to use it', url: `${DOCS}/studio/manual.md`, icon: 'book' }, + { label: 'Tutorial: one recipe, two PDFs', description: 'Read two different PDFs into one output, step by step', url: `${DOCS}/studio/pdf-agreements.md`, icon: 'mortar-board' }, + { label: 'Quick guide to recipes', description: 'What an input and an output recipe are, in ten minutes', url: `${DOCS}/recipes/quick-guide.md`, icon: 'zap' }, + { label: 'Example: vehicles', description: 'Cars from a configurator and a brochure, into one shape', url: `${DOCS}/recipes/vehicles-example.md`, icon: 'symbol-event' }, + { label: 'Example: movies', description: 'A movie database API into an output recipe', url: `${DOCS}/recipes/movies-example.md`, icon: 'symbol-event' }, + { label: 'Authoring guide', description: 'Every step, selector and transform a recipe can use', url: `${DOCS}/recipes/authoring.md`, icon: 'file-code' }, + { label: 'How it works', description: 'Recipes, pages, documents, mapping and running, explained', url: `${DOCS}/how-it-works/README.md`, icon: 'lightbulb' }, + { label: 'Ask or report a problem', description: 'Questions, ideas and bugs on GitHub', url: 'https://github.com/Open-Craw/OpenCraw/issues', icon: 'comment-discussion' }, +] diff --git a/apps/vscode-extension/src/learn-studio/learn-tree.client.ts b/apps/vscode-extension/src/learn-studio/learn-tree.client.ts new file mode 100644 index 00000000..8f2600d1 --- /dev/null +++ b/apps/vscode-extension/src/learn-studio/learn-tree.client.ts @@ -0,0 +1,20 @@ +import * as vscode from 'vscode' +import { LEARN_LINKS } from './learn-links.config' +import type { LearnLink } from './learn-links.config' + +/** The Learn view: a short list of where to read, each opening in the browser. */ +export class LearnTree implements vscode.TreeDataProvider { + getChildren (link?: LearnLink): LearnLink[] { + return link === undefined ? [...LEARN_LINKS] : [] + } + + getTreeItem (link: LearnLink): vscode.TreeItem { + const item = new vscode.TreeItem(link.label, vscode.TreeItemCollapsibleState.None) + item.description = link.description + item.tooltip = `${link.description}\n${link.url}` + item.iconPath = new vscode.ThemeIcon(link.icon) + item.command = { command: 'opencraw.openLink', title: 'Open', arguments: [link.url] } + + return item + } +} diff --git a/apps/vscode-extension/src/recipe-explorer/find-recipes.client.ts b/apps/vscode-extension/src/recipe-explorer/find-recipes.client.ts new file mode 100644 index 00000000..9853853e --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/find-recipes.client.ts @@ -0,0 +1,54 @@ +import * as vscode from 'vscode' +import { readRecipeHeader } from './read-recipe-header.algorithm' +import type { RecipeHeader } from './recipe-header.contract' + +/** The file names recipes go by: the `ocw-in`/`ocw-out` the schemas are bound to, and the `input`/`output` the example recipes use. */ +const RECIPE_FILES = '**/*.{ocw-in,ocw-out,input,output}.json' +const NOT_RECIPES = '**/{node_modules,dist,.git,out-tsc}/**' +/** A burst of saves (a branch switch, a formatter) is one refresh. */ +const REFRESH_DELAY_MS = 300 + +/** The recipes of the workspace, kept current. */ +export interface RecipeIndex { + /** Every recipe file in the workspace folders, who each one is. */ + headers: () => Promise + /** Fires when a recipe file was created, changed or deleted. */ + onDidChange: vscode.Event + dispose: () => void +} + +/** + * Finds the recipe files in the workspace and watches them. + * + * @returns The index. + */ +export function createRecipeIndex (): RecipeIndex { + const changed = new vscode.EventEmitter() + const watcher = vscode.workspace.createFileSystemWatcher(RECIPE_FILES) + let timer: NodeJS.Timeout | undefined + const schedule = (): void => { + clearTimeout(timer) + timer = setTimeout(() => { changed.fire() }, REFRESH_DELAY_MS) + } + const subscriptions = [watcher, watcher.onDidCreate(schedule), watcher.onDidChange(schedule), watcher.onDidDelete(schedule), vscode.workspace.onDidChangeWorkspaceFolders(schedule), changed] + + return { + headers: async () => { + const uris = await vscode.workspace.findFiles(RECIPE_FILES, NOT_RECIPES) + const headers = await Promise.all(uris.map(async (uri) => { + try { + return readRecipeHeader(uri.fsPath, new TextDecoder().decode(await vscode.workspace.fs.readFile(uri))) + } catch { + return + } + })) + + return headers.filter((header): header is RecipeHeader => header !== undefined) + }, + onDidChange: changed.event, + dispose: () => { + clearTimeout(timer) + for (const subscription of subscriptions) subscription.dispose() + }, + } +} diff --git a/apps/vscode-extension/src/recipe-explorer/group-by-output.algorithm.test.ts b/apps/vscode-extension/src/recipe-explorer/group-by-output.algorithm.test.ts new file mode 100644 index 00000000..f536f64b --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/group-by-output.algorithm.test.ts @@ -0,0 +1,29 @@ +import { groupByOutput } from './group-by-output.algorithm' +import type { readRecipeHeader } from './read-recipe-header.algorithm' + +const out = (id: string): ReturnType => ({ file: `/w/${id}.ocw-out.json`, kind: 'output', id }) +const inp = (id: string, output?: string): ReturnType => ({ file: `/w/${id}.ocw-in.json`, kind: 'input', id, ...(output !== undefined && { output }) }) + +describe('groupByOutput', () => { + it('puts each input under the output it names, groups and inputs in order', () => { + const groups = groupByOutput([inp('renault', 'offers'), out('vehicles'), inp('kia', 'vehicles'), out('offers'), inp('byd', 'vehicles')].map(header => header as NonNullable)) + + expect(groups.map(group => [group.outputId, group.outputFile, group.inputs.map(input => input.id)])).toEqual([ + ['offers', '/w/offers.ocw-out.json', ['renault']], + ['vehicles', '/w/vehicles.ocw-out.json', ['byd', 'kia']], + ]) + }) + + it('shows an output nothing feeds yet, and an input whose output is not in the workspace', () => { + const groups = groupByOutput([out('empty'), inp('lost', 'elsewhere')].map(header => header as NonNullable)) + + expect(groups).toEqual([ + { outputId: 'elsewhere', inputs: [{ file: '/w/lost.ocw-in.json', kind: 'input', id: 'lost', output: 'elsewhere' }] }, + { outputId: 'empty', outputFile: '/w/empty.ocw-out.json', inputs: [] }, + ]) + }) + + it('leaves out an input that names no output', () => { + expect(groupByOutput([inp('free')].map(header => header as NonNullable))).toEqual([]) + }) +}) diff --git a/apps/vscode-extension/src/recipe-explorer/group-by-output.algorithm.ts b/apps/vscode-extension/src/recipe-explorer/group-by-output.algorithm.ts new file mode 100644 index 00000000..72f04560 --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/group-by-output.algorithm.ts @@ -0,0 +1,27 @@ +import type { RecipeGroup, RecipeHeader } from './recipe-header.contract' + +/** + * Groups recipes the way a person thinks of them: an output recipe, and under it the input recipes that fill it. Groups are ordered by output id and + * so are the inputs of each. An output with no input still shows (nothing feeds it yet); an input that names an output the workspace does not hold + * is grouped under that name with no output file, so it is not lost; an input that names no output at all is left out (it cannot run: there is + * nothing to map its records onto). + * + * @param headers - Every recipe found. + * @returns The groups. + */ +export function groupByOutput (headers: readonly RecipeHeader[]): RecipeGroup[] { + const groups = new Map() + for (const header of headers) { + if (header.kind === 'output') groups.set(header.id, { outputId: header.id, outputFile: header.file, inputs: groups.get(header.id)?.inputs ?? [] }) + } + for (const header of headers) { + if (header.kind !== 'input' || header.output === undefined) continue + const group = groups.get(header.output) ?? { outputId: header.output, inputs: [] } + group.inputs.push(header) + groups.set(header.output, group) + } + + const ordered = Array.from(groups.values(), group => ({ ...group, inputs: [...group.inputs].sort((a, b) => a.id.localeCompare(b.id)) })) + + return ordered.sort((a, b) => a.outputId.localeCompare(b.outputId)) +} diff --git a/apps/vscode-extension/src/recipe-explorer/index.ts b/apps/vscode-extension/src/recipe-explorer/index.ts new file mode 100644 index 00000000..ecc42313 --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/index.ts @@ -0,0 +1,7 @@ +export { createRecipeIndex } from './find-recipes.client' +export type { RecipeIndex } from './find-recipes.client' +export { RecipeTree } from './recipe-tree.client' +export { groupByOutput } from './group-by-output.algorithm' +export { readRecipeHeader } from './read-recipe-header.algorithm' +export type { RecipeGroup, RecipeHeader } from './recipe-header.contract' +export type { RecipeNode } from './recipe-node.contract' diff --git a/apps/vscode-extension/src/recipe-explorer/read-recipe-header.algorithm.test.ts b/apps/vscode-extension/src/recipe-explorer/read-recipe-header.algorithm.test.ts new file mode 100644 index 00000000..2f30a63e --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/read-recipe-header.algorithm.test.ts @@ -0,0 +1,20 @@ +import { readRecipeHeader } from './read-recipe-header.algorithm' + +describe('readRecipeHeader', () => { + it('recognises an output recipe and an input recipe with the output it feeds', () => { + expect(readRecipeHeader('/w/cars.ocw-out.json', '{"kind":"output","id":"cars","version":1,"fields":{}}')).toEqual({ file: '/w/cars.ocw-out.json', kind: 'output', id: 'cars' }) + expect(readRecipeHeader('/w/kia.ocw-in.json', '{"kind":"input","id":"kia","output":"cars","steps":[]}')).toEqual({ file: '/w/kia.ocw-in.json', kind: 'input', id: 'kia', output: 'cars' }) + }) + + it('leaves out what is not a recipe: other JSON, text that is not JSON, a recipe with no id', () => { + expect(readRecipeHeader('/w/package.json', '{"name":"x","version":"1.0.0"}')).toBeUndefined() + expect(readRecipeHeader('/w/notes.json', 'not json')).toBeUndefined() + expect(readRecipeHeader('/w/list.json', '[1,2]')).toBeUndefined() + expect(readRecipeHeader('/w/a.ocw-in.json', '{"kind":"input","output":"cars"}')).toBeUndefined() + expect(readRecipeHeader('/w/a.ocw-in.json', '{"kind":"input","id":""}')).toBeUndefined() + }) + + it('reads an input that names no output as an input with none', () => { + expect(readRecipeHeader('/w/a.ocw-in.json', '{"kind":"input","id":"a"}')).toEqual({ file: '/w/a.ocw-in.json', kind: 'input', id: 'a' }) + }) +}) diff --git a/apps/vscode-extension/src/recipe-explorer/read-recipe-header.algorithm.ts b/apps/vscode-extension/src/recipe-explorer/read-recipe-header.algorithm.ts new file mode 100644 index 00000000..d24337b5 --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/read-recipe-header.algorithm.ts @@ -0,0 +1,24 @@ +import type { RecipeHeader } from './recipe-header.contract' + +/** + * Reads who a recipe file is from its text, looking only at `kind`, `id` and (for an input) `output`: the extension lists recipes without carrying the + * engine, so a file is not validated here, only recognised. A file that is not JSON, or is JSON of another kind (a package.json, a tsconfig), is not a recipe. + * + * @param file - The file's path. + * @param text - The file's content. + * @returns The header, or `undefined` when the file is not a recipe. + */ +export function readRecipeHeader (file: string, text: string): RecipeHeader | undefined { + let value: unknown + try { + value = JSON.parse(text) + } catch { + return undefined + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined + const { kind, id, output } = value as Record + if (typeof id !== 'string' || id === '' || (kind !== 'input' && kind !== 'output')) return undefined + if (kind === 'output') return { file, kind, id } + + return { file, kind, id, ...(typeof output === 'string' && output !== '' && { output }) } +} diff --git a/apps/vscode-extension/src/recipe-explorer/recipe-header.contract.ts b/apps/vscode-extension/src/recipe-explorer/recipe-header.contract.ts new file mode 100644 index 00000000..6bf774f8 --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/recipe-header.contract.ts @@ -0,0 +1,18 @@ +/** What the recipes view needs to know about a recipe file: who it is and, for an input, which output it feeds. */ +export interface RecipeHeader { + /** The file's absolute path. */ + file: string + kind: 'input' | 'output' + id: string + /** An input recipe's output id: the output recipe it maps its records onto. */ + output?: string +} + +/** An output recipe and the input recipes that feed it: one group of the recipes view. */ +export interface RecipeGroup { + /** The output id the inputs name. */ + outputId: string + /** The output recipe's file, when the workspace holds it; an input may name an output that is not there. */ + outputFile?: string + inputs: RecipeHeader[] +} diff --git a/apps/vscode-extension/src/recipe-explorer/recipe-node.contract.ts b/apps/vscode-extension/src/recipe-explorer/recipe-node.contract.ts new file mode 100644 index 00000000..d4b7f7e2 --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/recipe-node.contract.ts @@ -0,0 +1,9 @@ +import type { RecipeGroup, RecipeHeader } from './recipe-header.contract' + +/** + * What one row of the recipes view stands for, and what the commands on it (Run, Open in Studio) are handed: an output recipe with its inputs, or one + * input recipe with the group it belongs to. + */ +export type RecipeNode = + | { type: 'group', group: RecipeGroup } | + { type: 'input', header: RecipeHeader, group: RecipeGroup } diff --git a/apps/vscode-extension/src/recipe-explorer/recipe-tree.client.ts b/apps/vscode-extension/src/recipe-explorer/recipe-tree.client.ts new file mode 100644 index 00000000..34a0fd08 --- /dev/null +++ b/apps/vscode-extension/src/recipe-explorer/recipe-tree.client.ts @@ -0,0 +1,66 @@ +import * as vscode from 'vscode' +import { groupByOutput } from './group-by-output.algorithm' +import type { RecipeIndex } from './find-recipes.client' +import type { RecipeGroup } from './recipe-header.contract' +import type { RecipeNode } from './recipe-node.contract' + +/** What a group row says after its name: how many inputs feed it, or what is missing. */ +function groupDescription (group: RecipeGroup): string { + if (group.outputFile === undefined) return 'no output recipe with this id' + if (group.inputs.length === 0) return 'no input recipe feeds it yet' + + return `${String(group.inputs.length)} input${group.inputs.length === 1 ? '' : 's'}` +} + +/** + * The recipes view: each output recipe with the input recipes that feed it under it. Running a group runs every input; running an input runs just that one. + * The context values tell `package.json` which inline buttons a row gets. + */ +export class RecipeTree implements vscode.TreeDataProvider, vscode.Disposable { + private readonly changed = new vscode.EventEmitter() + private readonly subscription: vscode.Disposable + readonly onDidChangeTreeData = this.changed.event + + constructor (private readonly index: RecipeIndex) { + this.subscription = index.onDidChange(() => { this.refresh() }) + } + + refresh (): void { + this.changed.fire(undefined) + } + + async getChildren (node?: RecipeNode): Promise { + if (node?.type === 'input') return [] + if (node?.type === 'group') return node.group.inputs.map(header => ({ type: 'input', header, group: node.group })) + + return groupByOutput(await this.index.headers()).map(group => ({ type: 'group', group })) + } + + getTreeItem (node: RecipeNode): vscode.TreeItem { + if (node.type === 'input') { + const item = new vscode.TreeItem(node.header.id, vscode.TreeItemCollapsibleState.None) + item.iconPath = new vscode.ThemeIcon('file-code') + item.resourceUri = vscode.Uri.file(node.header.file) + item.tooltip = node.header.file + item.contextValue = node.group.outputFile === undefined ? 'opencraw.input.orphan' : 'opencraw.input' + item.command = { command: 'vscode.open', title: 'Open the recipe', arguments: [vscode.Uri.file(node.header.file)] } + + return item + } + const { group } = node + const item = new vscode.TreeItem(group.outputId, vscode.TreeItemCollapsibleState.Expanded) + item.description = groupDescription(group) + item.iconPath = new vscode.ThemeIcon(group.outputFile === undefined ? 'warning' : 'symbol-interface') + item.tooltip = group.outputFile ?? `No output recipe with the id "${group.outputId}" in this workspace` + // The context value says which buttons the row gets (`package.json`): Run needs an output recipe and at least one input; a group with no output recipe has nothing to open. + item.contextValue = group.outputFile === undefined ? 'opencraw.output.missing' : (group.inputs.length > 0 ? 'opencraw.output.runnable' : 'opencraw.output') + if (group.outputFile !== undefined) item.command = { command: 'vscode.open', title: 'Open the output recipe', arguments: [vscode.Uri.file(group.outputFile)] } + + return item + } + + dispose (): void { + this.subscription.dispose() + this.changed.dispose() + } +} diff --git a/apps/vscode-extension/src/recipe-run/index.ts b/apps/vscode-extension/src/recipe-run/index.ts new file mode 100644 index 00000000..785c0d9d --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/index.ts @@ -0,0 +1,5 @@ +export { createRecipeRunner } from './run-recipes.use-case' +export type { RecipeRunner, RecipeRunnerOptions } from './run-recipes.use-case' +export { planRun, runArguments } from './run-plan.algorithm' +export type { RunPlan, RunPlanResult } from './run-plan.algorithm' +export { SAVE_CHOICES, saveFilters, saveNameProblem } from './save-options.policy' diff --git a/apps/vscode-extension/src/recipe-run/run-cli.client.test.ts b/apps/vscode-extension/src/recipe-run/run-cli.client.test.ts new file mode 100644 index 00000000..d23ab6b8 --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/run-cli.client.test.ts @@ -0,0 +1,45 @@ +import { mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { cliScript, runCli } from './run-cli.client' + +async function script (body: string): Promise { + const directory = await mkdtemp(join(tmpdir(), 'run-cli-')) + const file = join(directory, 'program.mjs') + await writeFile(file, body) + + return file +} + +describe('runCli', () => { + it('runs the program with its arguments, hands out its error output as it comes and reports the exit code', async () => { + const file = await script('console.error("crawling " + process.argv.slice(2).join(" ")); process.exit(3)') + const chunks: string[] = [] + + const result = await runCli(file, ['a', 'b'], { onOutput: (chunk) => { chunks.push(chunk) } }) + + expect(result).toMatchObject({ code: 3, cancelled: false }) + expect(result.stderr).toContain('crawling a b') + expect(chunks.join('')).toContain('crawling a b') + }) + + it('stops the program when the signal aborts, and says it was stopped', async () => { + const file = await script('console.error("started"); setInterval(() => {}, 1000)') + const controller = new AbortController() + + const running = runCli(file, [], { signal: controller.signal, onOutput: () => { controller.abort() } }) + const result = await running + + expect(result.cancelled).toBe(true) + }) +}) + +describe('cliScript', () => { + it('finds the program in the repository checkout, beside the app', () => { + expect(cliScript(join(__dirname, '..', '..'))).toMatch(/packages[\\/]cli[\\/]bin[\\/]opencraw\.mjs$/) + }) + + it('says what is wrong when the program is nowhere to be found', () => { + expect(() => cliScript(join(tmpdir(), 'nowhere'))).toThrow('missing from this installation') + }) +}) diff --git a/apps/vscode-extension/src/recipe-run/run-cli.client.ts b/apps/vscode-extension/src/recipe-run/run-cli.client.ts new file mode 100644 index 00000000..ecc9c6d1 --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/run-cli.client.ts @@ -0,0 +1,71 @@ +import { fork } from 'node:child_process' +import { existsSync } from 'node:fs' +import { join } from 'node:path' + +/** What a run of the command line came to. */ +export interface CliResult { + /** The exit code; `null` when it was stopped before it ended. */ + code: number | null + /** Everything it wrote to its error output: the crawl's summary and its errors. */ + stderr: string + cancelled: boolean +} + +export interface CliOptions { + /** Called with each piece of error output as it arrives. */ + onOutput?: (chunk: string) => void + /** Aborting stops the process. */ + signal?: AbortSignal +} + +/** + * The command line program the extension carries: in the installed extension it sits in `node_modules`, in a checkout of the repository beside the + * app. + * + * @param extensionPath - Where the extension is installed. + * @returns The script to run. + * @throws Error when it is in neither place. + */ +export function cliScript (extensionPath: string): string { + const candidates = [ + join(extensionPath, 'node_modules', '@opencraw', 'cli', 'bin', 'opencraw.mjs'), + join(extensionPath, '..', '..', 'packages', 'cli', 'bin', 'opencraw.mjs'), + ] + const found = candidates.find(candidate => existsSync(candidate)) + if (found === undefined) throw new Error('the OpenCraw command line program is missing from this installation: reinstall the extension') + + return found +} + +/** + * Runs the command line with the editor's own executable as plain Node (`ELECTRON_RUN_AS_NODE`, as the Studio process does), so the extension ships no + * Node and the editor never waits on a crawl. + * + * @param script - The program. + * @param args - Its arguments. + * @param options - Output listener and cancellation. + * @returns How it ended. + */ +export async function runCli (script: string, args: readonly string[], options: CliOptions = {}): Promise { + return await new Promise((resolve, reject) => { + const child = fork(script, [...args], { env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }, stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + let stderr = '' + let cancelled = false + const stop = (): void => { + cancelled = true + child.kill() + } + child.stderr?.on('data', (chunk: Uint8Array | string) => { + const text = String(chunk) + stderr += text + options.onOutput?.(text) + }) + options.signal?.addEventListener('abort', stop, { once: true }) + if (options.signal?.aborted === true) stop() + child.on('error', reject) + child.on('exit', (code) => { + options.signal?.removeEventListener('abort', stop) + resolve({ code, stderr, cancelled }) + }) + }) +} diff --git a/apps/vscode-extension/src/recipe-run/run-plan.algorithm.test.ts b/apps/vscode-extension/src/recipe-run/run-plan.algorithm.test.ts new file mode 100644 index 00000000..1b9ca6a8 --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/run-plan.algorithm.test.ts @@ -0,0 +1,42 @@ +import type { RecipeNode } from '../recipe-explorer' +import { planRun, runArguments } from './run-plan.algorithm' + +const GROUP = { + outputId: 'vehicles', + outputFile: '/w/vehicles.ocw-out.json', + inputs: [ + { file: '/w/kia.ocw-in.json', kind: 'input' as const, id: 'kia', output: 'vehicles' }, + { file: '/w/byd.ocw-in.json', kind: 'input' as const, id: 'byd', output: 'vehicles' }, + ], +} + +describe('planRun', () => { + it('on an input recipe runs just that one, with the output it maps onto', () => { + const node: RecipeNode = { type: 'input', header: GROUP.inputs[0], group: GROUP } + + expect(planRun(node)).toEqual({ ok: true, plan: { paths: ['/w/vehicles.ocw-out.json', '/w/kia.ocw-in.json'], only: ['kia'], outputId: 'vehicles', output: '/w/vehicles.ocw-out.json', label: 'kia' } }) + }) + + it('on an output recipe runs every input that feeds it', () => { + const result = planRun({ type: 'group', group: GROUP }) + + expect(result).toMatchObject({ ok: true, plan: { paths: ['/w/vehicles.ocw-out.json', '/w/kia.ocw-in.json', '/w/byd.ocw-in.json'], only: [], label: 'vehicles (2 inputs)' } }) + }) + + it('says why there is nothing to run: no output recipe, or no input', () => { + const orphan = { outputId: 'lost', inputs: [GROUP.inputs[0]] } + expect(planRun({ type: 'group', group: orphan })).toMatchObject({ ok: false, reason: expect.stringContaining('no output recipe with the id "lost"') }) + expect(planRun({ type: 'input', header: GROUP.inputs[0], group: orphan })).toMatchObject({ ok: false }) + expect(planRun({ type: 'group', group: { ...GROUP, inputs: [] } })).toMatchObject({ ok: false, reason: expect.stringContaining('No input recipe feeds "vehicles"') }) + }) +}) + +describe('runArguments', () => { + it('writes the records to the given JSON Lines file, one input at a time when asked, with the hooks module when set', () => { + const planned = planRun({ type: 'input', header: GROUP.inputs[1], group: GROUP }) + if (!planned.ok) throw new Error('expected a plan') + + expect(runArguments(planned.plan, '/tmp/run.jsonl')).toEqual(['run', '/w/vehicles.ocw-out.json', '/w/byd.ocw-in.json', '--out', '/tmp/run.jsonl', '--trace', '--only', 'byd']) + expect(runArguments(planned.plan, '/tmp/run.jsonl', '/h/hooks.mjs').slice(-2)).toEqual(['--hooks', '/h/hooks.mjs']) + }) +}) diff --git a/apps/vscode-extension/src/recipe-run/run-plan.algorithm.ts b/apps/vscode-extension/src/recipe-run/run-plan.algorithm.ts new file mode 100644 index 00000000..8c5f09e7 --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/run-plan.algorithm.ts @@ -0,0 +1,46 @@ +import type { RecipeNode } from '../recipe-explorer' + +/** What to run, as the command line needs it. */ +export interface RunPlan { + /** The output recipe, then the input recipes: the paths `opencraw run` is given. */ + paths: string[] + /** The input recipe ids to run (`--only`); empty runs every input given. */ + only: string[] + outputId: string + /** The output recipe's file: `opencraw export` needs it to name the columns. */ + output: string + /** What the run is called to the person. */ + label: string +} + +export type RunPlanResult = { ok: true, plan: RunPlan } | { ok: false, reason: string } + +/** + * Decides what a Run click runs: on an input recipe, that one input; on an output recipe, every input that feeds it. Both need the output recipe, + * which is what the records are mapped onto. + * + * @param node - The row that was clicked. + * @returns The plan, or why there is nothing to run. + */ +export function planRun (node: RecipeNode): RunPlanResult { + const { group } = node + if (group.outputFile === undefined) return { ok: false, reason: `There is no output recipe with the id "${group.outputId}" in this workspace, so there is nothing to map the records onto.` } + if (node.type === 'input') { + return { ok: true, plan: { paths: [group.outputFile, node.header.file], only: [node.header.id], outputId: group.outputId, output: group.outputFile, label: node.header.id } } + } + if (group.inputs.length === 0) return { ok: false, reason: `No input recipe feeds "${group.outputId}" yet.` } + + return { ok: true, plan: { paths: [group.outputFile, ...group.inputs.map(input => input.file)], only: [], outputId: group.outputId, output: group.outputFile, label: `${group.outputId} (${String(group.inputs.length)} input${group.inputs.length === 1 ? '' : 's'})` } } +} + +/** + * The command line that runs a plan, its records going to a JSON Lines file: that format streams, and it is what `export` reads afterwards. + * + * @param plan - What to run. + * @param records - The JSON Lines file to write. + * @param hooks - The person's hooks module, when they set one. + * @returns The arguments after the program name. + */ +export function runArguments (plan: RunPlan, records: string, hooks?: string): string[] { + return ['run', ...plan.paths, '--out', records, '--trace', ...plan.only.flatMap(id => ['--only', id]), ...(hooks === undefined ? [] : ['--hooks', hooks])] +} diff --git a/apps/vscode-extension/src/recipe-run/run-progress.algorithm.test.ts b/apps/vscode-extension/src/recipe-run/run-progress.algorithm.test.ts new file mode 100644 index 00000000..e95fa4a9 --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/run-progress.algorithm.test.ts @@ -0,0 +1,22 @@ +import { progressMessage, recordCount } from './run-progress.algorithm' + +describe('progressMessage', () => { + it('is the last non-empty line the run wrote, trimmed', () => { + expect(progressMessage('first\n GET https://x.test/a \n\n')).toBe('GET https://x.test/a') + expect(progressMessage('only line')).toBe('only line') + }) + + it('shortens a long line and says nothing for an empty chunk', () => { + const message = progressMessage('x'.repeat(200)) + expect(message).toHaveLength(90) + expect(message?.endsWith('…')).toBe(true) + expect(progressMessage('\n \n')).toBeUndefined() + }) +}) + +describe('recordCount', () => { + it('counts the non-empty lines of a JSON Lines file', () => { + expect(recordCount('{"a":1}\n{"a":2}\n\n')).toBe(2) + expect(recordCount('')).toBe(0) + }) +}) diff --git a/apps/vscode-extension/src/recipe-run/run-progress.algorithm.ts b/apps/vscode-extension/src/recipe-run/run-progress.algorithm.ts new file mode 100644 index 00000000..f1aa400e --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/run-progress.algorithm.ts @@ -0,0 +1,26 @@ +/** The longest a progress message is, so a notification does not grow wide. */ +const MESSAGE_LIMIT = 90 + +/** + * The last thing the run said, as a short line for the progress notification: the crawl's trace lines, one per event, tell what it is doing now. + * + * @param chunk - What the run wrote to its error output since the last call. + * @returns The last non-empty line, shortened; `undefined` when the chunk has none. + */ +export function progressMessage (chunk: string): string | undefined { + const lines = chunk.split(/\r?\n/).map(line => line.trim()).filter(line => line !== '') + const last = lines.at(-1) + if (last === undefined) return undefined + + return last.length > MESSAGE_LIMIT ? `${last.slice(0, MESSAGE_LIMIT - 1)}…` : last +} + +/** + * How many records a JSON Lines file holds. + * + * @param text - The file's content. + * @returns The number of non-empty lines. + */ +export function recordCount (text: string): number { + return text.split('\n').filter(line => line.trim() !== '').length +} diff --git a/apps/vscode-extension/src/recipe-run/run-recipes.use-case.ts b/apps/vscode-extension/src/recipe-run/run-recipes.use-case.ts new file mode 100644 index 00000000..dd86d1a9 --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/run-recipes.use-case.ts @@ -0,0 +1,147 @@ +import { copyFile, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, dirname, extname, join } from 'node:path' +import * as vscode from 'vscode' +import type { RecipeNode } from '../recipe-explorer' +import { cliScript, runCli } from './run-cli.client' +import type { CliResult } from './run-cli.client' +import { progressMessage, recordCount } from './run-progress.algorithm' +import { planRun, runArguments } from './run-plan.algorithm' +import type { RunPlan } from './run-plan.algorithm' +import { defaultFileName, saveFilters, saveNameProblem } from './save-options.policy' + +/** Runs recipes from the recipes view and saves the result where the person says. */ +export interface RecipeRunner { + /** Runs the input recipe, or every input of the output recipe, that was clicked; asks where to save when the crawl is over. */ + run: (node: RecipeNode) => Promise +} + +export interface RecipeRunnerOptions { + extensionPath: string + /** Where the crawl's own messages go, for a person who wants to read them. */ + output: vscode.OutputChannel + /** The hooks module the person set, read when a run starts so a changed setting counts at once. */ + hooksFile: () => string | undefined +} + +/** + * Asks where to save, with the operating system's own dialog. The extension of the name is the format, so a name that is not one is explained and the + * dialog opens again. + * + * @param plan - What was run, for the suggested name. + * @returns The file to write, or `undefined` when the dialog was closed. + */ +async function askWhereToSave (plan: RunPlan): Promise { + const folder = vscode.workspace.workspaceFolders?.[0]?.uri ?? vscode.Uri.file(dirname(plan.output)) + let suggestion = vscode.Uri.joinPath(folder, defaultFileName(plan.outputId)) + for (;;) { + const picked = await vscode.window.showSaveDialog({ title: `Save the ${plan.outputId} records as…`, saveLabel: 'Save', defaultUri: suggestion, filters: saveFilters() }) + if (picked === undefined) return undefined + const problem = saveNameProblem(picked.fsPath) + if (problem === undefined) return picked.fsPath + await vscode.window.showWarningMessage(problem, { modal: true }) + suggestion = vscode.Uri.joinPath(folder, `${basename(picked.fsPath, extname(picked.fsPath))}.xlsx`) + } +} + +/** Shows the saved file: text formats in the editor, the others (a spreadsheet, a page, a PDF) in the program that opens them. */ +async function showSaved (path: string): Promise { + const uri = vscode.Uri.file(path) + if (['.xlsx', '.html', '.pdf'].includes(extname(path).toLowerCase())) await vscode.env.openExternal(uri) + else await vscode.commands.executeCommand('vscode.open', uri) +} + +/** How many records the crawl wrote; none when it wrote nothing at all (no file). */ +async function collected (records: string): Promise { + try { + return recordCount(await readFile(records, 'utf8')) + } catch { + return 0 + } +} + +/** + * The recipes view's Run: crawl in a process of its own with a notification that can cancel it, then, when it is over, ask where to save and in what + * format (the file's extension), and write the records there. The crawl goes to a temporary JSON Lines file first, so the question comes after the + * work, when the person knows what came out. + * + * @param options - The extension's folder, the output channel, the hooks setting. + * @returns The runner. + */ +export function createRecipeRunner (options: RecipeRunnerOptions): RecipeRunner { + const script = (): string => cliScript(options.extensionPath) + + async function crawl (plan: RunPlan, records: string): Promise { + const args = runArguments(plan, records, options.hooksFile()) + options.output.appendLine(`> opencraw ${args.join(' ')}`) + + return await vscode.window.withProgress({ location: vscode.ProgressLocation.Notification, title: `OpenCraw: running ${plan.label}`, cancellable: true }, async (progress, token) => { + const controller = new AbortController() + token.onCancellationRequested(() => { controller.abort() }) + + return await runCli(script(), args, { + signal: controller.signal, + onOutput: (chunk) => { + options.output.append(chunk) + const message = progressMessage(chunk) + if (message !== undefined) progress.report({ message }) + }, + }) + }) + } + + async function save (plan: RunPlan, records: string, target: string): Promise { + if (extname(target).toLowerCase() === '.jsonl') { + await copyFile(records, target) + + return + } + const result = await runCli(script(), ['export', records, plan.output, '--out', target]) + if (result.code !== 0) throw new Error(result.stderr.trim().split('\n').pop() ?? 'the export failed') + } + + async function finish (plan: RunPlan, records: string, result: CliResult): Promise { + if (result.cancelled) { + await vscode.window.showInformationMessage(`OpenCraw: ${plan.label} was stopped.`) + + return + } + const count = await collected(records) + if (count === 0) { + const choice = await vscode.window.showErrorMessage(`OpenCraw: ${plan.label} produced no records${result.code === 0 ? '' : ' (it stopped with an error)'}.`, 'Show the details') + if (choice !== undefined) options.output.show() + + return + } + const target = await askWhereToSave(plan) + if (target === undefined) { + await vscode.window.showInformationMessage(`OpenCraw: ${String(count)} records were collected but not saved.`) + + return + } + await save(plan, records, target) + const partial = result.code === 0 ? '' : ' Some recipes stopped with an error: see the OpenCraw output.' + const choice = await vscode.window.showInformationMessage(`OpenCraw: ${String(count)} records saved to ${basename(target)}.${partial}`, 'Open', 'Show in folder') + if (choice === 'Open') await showSaved(target) + else if (choice === 'Show in folder') await vscode.commands.executeCommand('revealFileInOS', vscode.Uri.file(target)) + } + + return { + run: async (node) => { + const planned = planRun(node) + if (!planned.ok) { + await vscode.window.showWarningMessage(`OpenCraw: ${planned.reason}`) + + return + } + const records = join(tmpdir(), `opencraw-run-${String(Date.now())}.jsonl`) + try { + await finish(planned.plan, records, await crawl(planned.plan, records)) + } catch (error) { + await vscode.window.showErrorMessage(`OpenCraw: ${error instanceof Error ? error.message : String(error)}`) + } finally { + await rm(records, { force: true }) + } + }, + } +} diff --git a/apps/vscode-extension/src/recipe-run/save-options.policy.test.ts b/apps/vscode-extension/src/recipe-run/save-options.policy.test.ts new file mode 100644 index 00000000..608eeaa3 --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/save-options.policy.test.ts @@ -0,0 +1,21 @@ +import { SAVE_CHOICES, defaultFileName, saveFilters, saveNameProblem } from './save-options.policy' + +describe('save options', () => { + it('offers every format the engine writes, one filter each, the spreadsheet first', () => { + expect(SAVE_CHOICES.map(choice => choice.extension)).toEqual(['xlsx', 'csv', 'html', 'pdf', 'json', 'jsonl', 'yml']) + expect(saveFilters()['Excel workbook (.xlsx)']).toEqual(['xlsx']) + expect(Object.keys(saveFilters())).toHaveLength(7) + expect(defaultFileName('vehicles')).toBe('vehicles.xlsx') + }) + + it('accepts a name whose extension is a format, in any case, and .yaml for YAML', () => { + expect(saveNameProblem('/out/cars.CSV')).toBeUndefined() + expect(saveNameProblem('/out/cars.yaml')).toBeUndefined() + expect(saveNameProblem(String.raw`C:\out\cars.pdf`)).toBeUndefined() + }) + + it('explains a name that is not one, saying which extensions work', () => { + expect(saveNameProblem('/out/cars.txt')).toMatch(/ends in "\.txt".*\.xlsx, \.csv, \.html, \.pdf, \.json, \.jsonl, \.yml/) + expect(saveNameProblem('/out/cars')).toContain('has no extension') + }) +}) diff --git a/apps/vscode-extension/src/recipe-run/save-options.policy.ts b/apps/vscode-extension/src/recipe-run/save-options.policy.ts new file mode 100644 index 00000000..fe960419 --- /dev/null +++ b/apps/vscode-extension/src/recipe-run/save-options.policy.ts @@ -0,0 +1,45 @@ +import { extname } from 'node:path' + +/** One format the person can save a run in, as the save dialog offers it. */ +export interface SaveChoice { + label: string + extension: string +} + +/** Every format the engine writes, best first for a person who wants to open the result: a spreadsheet, a page, a PDF, then the data formats. */ +export const SAVE_CHOICES: readonly SaveChoice[] = [ + { label: 'Excel workbook', extension: 'xlsx' }, + { label: 'CSV', extension: 'csv' }, + { label: 'Web page (HTML)', extension: 'html' }, + { label: 'PDF', extension: 'pdf' }, + { label: 'JSON', extension: 'json' }, + { label: 'JSON Lines', extension: 'jsonl' }, + { label: 'YAML', extension: 'yml' }, +] + +const EXTENSIONS = new Set([...SAVE_CHOICES.map(choice => `.${choice.extension}`), '.yaml']) + +/** The filters of the operating system's save dialog: one per format. */ +export function saveFilters (): Record { + return Object.fromEntries(SAVE_CHOICES.map(choice => [`${choice.label} (.${choice.extension})`, [choice.extension]])) +} + +/** The file name the dialog starts with: the output's id in the first format offered. */ +export function defaultFileName (outputId: string): string { + return `${outputId}.${SAVE_CHOICES[0].extension}` +} + +/** + * Checks the file the person named: its extension is the format, so it has to be one the engine writes. + * + * @param path - What the save dialog returned. + * @returns `undefined` when the name is fine, else what to tell the person. + */ +export function saveNameProblem (path: string): string | undefined { + const extension = extname(path).toLowerCase() + if (EXTENSIONS.has(extension)) return undefined + const shown = extension === '' ? 'has no extension' : `ends in "${extension}"` + const choices = SAVE_CHOICES.map(choice => `.${choice.extension}`).join(', ') + + return `"${path}" ${shown}. The extension is the format: name the file ${choices} (or .yaml).` +} diff --git a/package-lock.json b/package-lock.json index f586cf3f..cd2581fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -117,7 +117,7 @@ "@opencraw/studio": "^0.0.1" }, "devDependencies": { - "@types/vscode": "1.90.0", + "@types/vscode": "1.125.0", "@vscode/test-electron": "^3.1.0" }, "engines": { @@ -14755,9 +14755,9 @@ "license": "MIT" }, "node_modules/@types/vscode": { - "version": "1.90.0", - "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.90.0.tgz", - "integrity": "sha512-oT+ZJL7qHS9Z8bs0+WKf/kQ27qWYR3trsXpq46YDjFqBsMLG4ygGGjPaJ2tyrH0wJzjOEmDyg9PDJBBhWg9pkQ==", + "version": "1.125.0", + "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.125.0.tgz", + "integrity": "sha512-0icm/ZQAaism87P0ekHqi4/Ju9du+Tm0RUW+y7vqRsxY2cY0FNRX1nAnaW7nT6npPt2tfHiheZ55Zm9UhqonFA==", "dev": true, "license": "MIT" }, diff --git a/packages/cli/src/arguments/command.contract.ts b/packages/cli/src/arguments/command.contract.ts index 94007178..cd97fe1e 100644 --- a/packages/cli/src/arguments/command.contract.ts +++ b/packages/cli/src/arguments/command.contract.ts @@ -18,6 +18,16 @@ export type Command = | { name: 'help' } | { name: 'version' } | { name: 'validate', paths: string[] } | + { + name: 'export' + /** A finished run's JSON Lines file. */ + records: string + /** The output recipe the records were mapped to: it names the columns. */ + output: string + /** The file to write; its extension is the format. */ + out: string + options: CommonOptions + } | { name: 'diff' previous: string diff --git a/packages/cli/src/arguments/parse-arguments.algorithm.test.ts b/packages/cli/src/arguments/parse-arguments.algorithm.test.ts index 7f836f73..7c5b6733 100644 --- a/packages/cli/src/arguments/parse-arguments.algorithm.test.ts +++ b/packages/cli/src/arguments/parse-arguments.algorithm.test.ts @@ -57,6 +57,9 @@ describe('parseArguments', () => { expect(() => parseArguments(['probe'])).toThrow('exactly one URL') expect(() => parseArguments(['run', 'a', '--bogus'])).toThrow(/Unknown option/) expect(() => parseArguments(['diff', 'a.jsonl'])).toThrow('diff needs two JSON Lines files') + expect(parseArguments(['export', 'run.jsonl', 'product.output.json', '--out', 'run.xlsx'])).toMatchObject({ name: 'export', records: 'run.jsonl', output: 'product.output.json', out: 'run.xlsx' }) + expect(() => parseArguments(['export', 'run.jsonl'])).toThrow('export needs the records file') + expect(() => parseArguments(['export', 'run.jsonl', 'product.output.json'])).toThrow('export needs --out') expect(() => parseArguments(['run', 'a', '--diff', 'x', '--resume', '--append', '--out', 'o'])).toThrow('use one or the other') expect(() => parseArguments(['run', 'a', '--changes', 'c'])).toThrow('--changes needs --diff') expect(() => parseArguments(['run', 'a', '--host-concurrency', '0'])).toThrow('--host-concurrency needs a whole number of at least 1, not "0"') diff --git a/packages/cli/src/arguments/parse-arguments.algorithm.ts b/packages/cli/src/arguments/parse-arguments.algorithm.ts index ecded029..251b0e5b 100644 --- a/packages/cli/src/arguments/parse-arguments.algorithm.ts +++ b/packages/cli/src/arguments/parse-arguments.algorithm.ts @@ -8,6 +8,9 @@ Commands run Crawl. Records go to --out (its extension picks the format), or to stdout as JSON Lines. probe Fetch a page and report where its data lives. diff Compare two runs' records by key: added, removed, changed. + export --out + Write a finished run's records again in another format (the extension + of --out chooses it, as for run). studio [folder] [--open ] Open a recipe folder in the browser: run samples, read the trace next to the page (needs @opencraw/studio). --open prints a link that opens a folder, recipe or @@ -116,6 +119,12 @@ export function parseArguments (argv: readonly string[], env: Record: its extension is the format') + + return { name: 'export', records: rest[0], output: rest[1], out: values.out, options } + } case 'diff': { if (rest.length !== 2) throw new Error('diff needs two JSON Lines files: the previous run, then the current one') diff --git a/packages/cli/src/export/export-records.use-case.test.ts b/packages/cli/src/export/export-records.use-case.test.ts new file mode 100644 index 00000000..c34645d2 --- /dev/null +++ b/packages/cli/src/export/export-records.use-case.test.ts @@ -0,0 +1,58 @@ +import { mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { exportRecords } from './export-records.use-case' + +const OUTPUT = { kind: 'output', id: 'cars', version: 1, fields: { model: { type: 'string' }, price: { type: 'number' } } } +const LINES = [ + { model: 'Clio', price: 19900, _source: { recipeId: 'shop', url: 'https://x.test/1', emittedAt: '2026-10-16T10:00:00.000Z' } }, + { model: 'Twingo', price: 17500, _source: { recipeId: 'shop', url: 'https://x.test/2', emittedAt: '2026-10-16T10:00:01.000Z' } }, +] + +function terminal (): { out: string[], err: string[], api: { out: (line: string) => void, err: (line: string) => void } } { + const out: string[] = [] + const err: string[] = [] + const toOut = (line: string): void => { + out.push(line) + } + const toErr = (line: string): void => { + err.push(line) + } + + return { out, err, api: { out: toOut, err: toErr } } +} + +async function files (): Promise<{ directory: string, records: string, output: string }> { + const directory = await mkdtemp(join(tmpdir(), 'export-cli-')) + const records = join(directory, 'run.jsonl') + const output = join(directory, 'cars.output.json') + await writeFile(records, `${LINES.map(line => JSON.stringify(line)).join('\n')}\n`) + await writeFile(output, JSON.stringify(OUTPUT)) + + return { directory, records, output } +} + +describe('exportRecords', () => { + it('writes the run again in the format the extension names, with the output recipe\'s fields as columns', async () => { + const { directory, records, output } = await files() + const out = join(directory, 'cars.csv') + const t = terminal() + + const code = await exportRecords({ name: 'export', records, output, out, options: { insecureTls: false } }, t.api) + + expect(code).toBe(0) + expect(await readFile(out, 'utf8')).toBe('model,price\r\nClio,19900\r\nTwingo,17500\r\n') + expect(t.err[0]).toMatch(/^2 records written to /) + }) + + it('refuses an extension that names no format, and a file that is not an output recipe', async () => { + const { directory, records, output } = await files() + const t = terminal() + + expect(await exportRecords({ name: 'export', records, output, out: join(directory, 'cars.txt'), options: { insecureTls: false } }, t.api)).toBe(1) + expect(t.err.join('\n')).toContain('ends in ".txt"') + + const bad = terminal() + expect(await exportRecords({ name: 'export', records, output: records, out: join(directory, 'cars.csv'), options: { insecureTls: false } }, bad.api)).toBe(1) + }) +}) diff --git a/packages/cli/src/export/export-records.use-case.ts b/packages/cli/src/export/export-records.use-case.ts new file mode 100644 index 00000000..0d7a6b83 --- /dev/null +++ b/packages/cli/src/export/export-records.use-case.ts @@ -0,0 +1,31 @@ +import { readFile } from 'node:fs/promises' +import { RecipeValidationError, fileSink, outputRecordOf, parseOutputRecipe, readRecordsFile } from '@opencraw/core' +import type { Command } from '../arguments' +import type { Terminal } from '../terminal' + +/** + * Writes a finished run's JSON Lines file again in another format: the extension of `--out` chooses it, as it does for `run`. The output recipe + * supplies the columns (its fields, in order, with their types and descriptions). + * + * @param command - The parsed `export` command. + * @param terminal - Where the result and errors go. + * @returns The exit code: 0, or 1 when a file cannot be read, the recipe is not an output recipe, or the extension names no format. + */ +export async function exportRecords (command: Extract, terminal: Terminal): Promise { + try { + const sink = fileSink(command.out, { browser: { executablePath: command.options.browserPath } }) + const output = parseOutputRecipe(JSON.parse(await readFile(command.output, 'utf8')) as unknown, command.output) + const stored = await readRecordsFile(command.records) + await sink.open(output) + for (const line of stored) await sink.write(outputRecordOf(line)) + const summary = await sink.close() + terminal.err(`${String(summary.written)} records written to ${summary.location ?? command.out}`) + + return 0 + } catch (error) { + if (error instanceof RecipeValidationError) for (const issue of error.issues) terminal.err(`${error.source}: ${issue.path === '' ? '(root)' : issue.path}: ${issue.message}`) + else terminal.err(error instanceof Error ? error.message : String(error)) + + return 1 + } +} diff --git a/packages/cli/src/export/index.ts b/packages/cli/src/export/index.ts new file mode 100644 index 00000000..8458233b --- /dev/null +++ b/packages/cli/src/export/index.ts @@ -0,0 +1 @@ +export { exportRecords } from './export-records.use-case' diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 202d9413..5bdacaa1 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,5 +1,6 @@ import { USAGE, parseArguments } from './arguments' import { diffFiles } from './diff' +import { exportRecords } from './export' import { probePage } from './probe' import { readFileSync } from 'node:fs' import { runRecipes } from './run' @@ -45,6 +46,8 @@ export async function main (argv: readonly string[], terminal: Terminal): Promis } case 'diff': { return diffFiles(command, terminal) } + case 'export': { return exportRecords(command, terminal) + } case 'studio': { return runStudio(command.folder, terminal, undefined, command.plugins, undefined, command.open) } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 992335c2..f3c70392 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -18,7 +18,7 @@ export type { FieldTrace, MappingTrace, OutputRecord } from './output-mapping' export { RecordRejectedError, MappingFailedError, resolveMissingPolicy, isMissing } from './output-mapping' export type { RecordSink, SinkSummary, MemorySink, DedupeScope } from './record-sink' export { memorySink, jsonLinesSink } from './record-sink' -export { fileSink, OUTPUT_EXTENSIONS, OUTPUT_FORMATS, outputFormatOf, UnsupportedOutputFormatError, tableOf, htmlReport } from './record-export' +export { fileSink, outputRecordOf, OUTPUT_EXTENSIONS, OUTPUT_FORMATS, outputFormatOf, UnsupportedOutputFormatError, tableOf, htmlReport } from './record-export' export type { FileSinkOptions, OutputFormat, ExportTable, TableCell, TableColumn, ReportOptions } from './record-export' export type { CrawlEvent, CrawlListener } from './crawl-events' export { traceLine } from './crawl-events' diff --git a/packages/core/src/record-export/index.ts b/packages/core/src/record-export/index.ts index ef338f58..4d1ae899 100644 --- a/packages/core/src/record-export/index.ts +++ b/packages/core/src/record-export/index.ts @@ -3,6 +3,7 @@ export type { FileSinkOptions } from './file-sink.repository' export { OUTPUT_EXTENSIONS, OUTPUT_FORMATS, outputFormatOf } from './output-format.policy' export type { OutputFormat } from './output-format.policy' export { UnsupportedOutputFormatError } from './unsupported-output-format.error' +export { outputRecordOf } from './stored-record.mapper' export { tableOf } from './table.mapper' export type { ExportTable, TableCell, TableColumn } from './table.mapper' export { htmlReport } from './html-report.algorithm' diff --git a/packages/core/src/record-export/record-export.test.ts b/packages/core/src/record-export/record-export.test.ts index 75df6e03..eccfdc71 100644 --- a/packages/core/src/record-export/record-export.test.ts +++ b/packages/core/src/record-export/record-export.test.ts @@ -9,6 +9,7 @@ import { csvText } from './csv.algorithm' import { fileSink } from './file-sink.repository' import { htmlReport } from './html-report.algorithm' import { outputFormatOf } from './output-format.policy' +import { outputRecordOf } from './stored-record.mapper' import { cellOf, tableOf } from './table.mapper' import { UnsupportedOutputFormatError } from './unsupported-output-format.error' import { xlsxBytes } from './xlsx.algorithm' @@ -198,3 +199,17 @@ describe('fileSink', () => { expect(bytes.length).toBeGreaterThan(2000) }, 60_000) }) + +describe('outputRecordOf', () => { + it('turns a JSON Lines line back into a record, with its source and key', () => { + expect(outputRecordOf({ model: 'Clio', price: 1, _source: { recipeId: 'shop', url: 'https://x.test', emittedAt: '2026-10-16T10:00:00.000Z' }, _key: '["Clio"]' })).toEqual({ + data: { model: 'Clio', price: 1 }, + key: '["Clio"]', + source: { recipeId: 'shop', url: 'https://x.test', emittedAt: '2026-10-16T10:00:00.000Z' }, + }) + }) + + it('gives a line with no source an empty one, and no key', () => { + expect(outputRecordOf({ model: 'Clio' })).toEqual({ data: { model: 'Clio' }, key: null, source: { recipeId: '', url: '', emittedAt: '' } }) + }) +}) diff --git a/packages/core/src/record-export/stored-record.mapper.ts b/packages/core/src/record-export/stored-record.mapper.ts new file mode 100644 index 00000000..2bcc2723 --- /dev/null +++ b/packages/core/src/record-export/stored-record.mapper.ts @@ -0,0 +1,24 @@ +import type { OutputRecord } from '../output-mapping' + +/** + * A record as a JSON Lines file holds it (the data, with the sink's `_source` and, when appending, `_key`) turned back into the record a sink takes, + * so a finished run's file can be written again in another format. + * + * @param stored - One line of a records file, parsed. + * @returns The record; a line with no `_source` gets an empty one. + */ +export function outputRecordOf (stored: Readonly>): OutputRecord { + const { _source: source, _key: key, ...data } = stored + const origin = typeof source === 'object' && source !== null ? source as Partial : {} + + return { + data, + key: typeof key === 'string' ? key : null, + source: { + recipeId: typeof origin.recipeId === 'string' ? origin.recipeId : '', + url: typeof origin.url === 'string' ? origin.url : '', + emittedAt: typeof origin.emittedAt === 'string' ? origin.emittedAt : '', + ...(typeof origin.item === 'string' && { item: origin.item }), + }, + } +}