From 9df41d384d1a2bbe40e69623fff50afc6abfeb4d Mon Sep 17 00:00:00 2001 From: zigai Date: Fri, 16 Oct 2026 00:29:39 +0200 Subject: [PATCH] feat(pi-glowup)!: format and color script previews --- extensions/pi-glowup/README.md | 30 +- extensions/pi-glowup/config.schema.json | 268 ++++++++++++++- extensions/pi-glowup/docs/script-previews.md | 45 ++- extensions/pi-glowup/scripts/build.mjs | 26 +- extensions/pi-glowup/src/pi/lifecycle.ts | 4 +- .../src/pi/patches/markdown-syntax.ts | 55 ++- .../pi-glowup/src/pi/syntax-lifecycle.ts | 17 +- .../src/rendering/diff/text-renderer.ts | 43 ++- .../src/rendering/syntax/code-component.ts | 54 ++- .../src/rendering/syntax/highlighter.ts | 237 +++++++++++-- .../src/rendering/syntax/tokenize-worker.ts | 53 +++ .../src/rendering/syntax/tokenizer.ts | 220 ++++++++++++ extensions/pi-glowup/src/settings-input.ts | 120 ++++++- .../pi-glowup/src/settings.prevalidated.ts | 24 +- .../src/tools/built-in/bash/command.ts | 137 ++++++-- .../tools/built-in/bash/display-profile.ts | 93 +++++ .../src/tools/built-in/bash/format-preview.ts | 26 +- .../src/tools/built-in/bash/formatter.ts | 118 +++++-- .../src/tools/built-in/bash/preview.ts | 241 +++++++++---- .../src/tools/built-in/bash/renderer.ts | 18 + .../tools/built-in/bash/script-renderer.ts | 205 ++++++++--- .../src/tools/built-in/bash/shell-words.ts | 85 +++++ .../pi-glowup/src/tools/external/codemode.ts | 324 +++++++++++++----- extensions/pi-glowup/src/tools/types.ts | 17 +- extensions/pi-glowup/test/config/load.test.ts | 16 +- extensions/pi-glowup/test/package.test.ts | 86 ++++- .../pi-glowup/test/pi/index-lifecycle.test.ts | 23 +- .../pi-glowup/test/pi/markdown-patch.test.ts | 82 ++++- extensions/pi-glowup/test/pty/pi-cli.test.ts | 66 ++-- .../test/rendering/rendering.test.ts | 5 +- .../syntax/syntax-code-component.test.ts | 193 ++++++++++- .../syntax-highlighter-lifecycle.test.ts | 29 +- .../test/rendering/syntax/syntax.test.ts | 189 +++++++--- .../tools/built-in/bash/bash-analysis.test.ts | 63 ++++ .../built-in/bash/bash-call-rendering.test.ts | 43 +++ .../built-in/bash/script-formatters.test.ts | 195 ++++++++--- .../bash/script-preview-events.test.ts | 190 +++++++++- .../test/tools/third-party-renderers.test.ts | 122 ++++++- 38 files changed, 3270 insertions(+), 492 deletions(-) create mode 100644 extensions/pi-glowup/src/rendering/syntax/tokenize-worker.ts create mode 100644 extensions/pi-glowup/src/rendering/syntax/tokenizer.ts create mode 100644 extensions/pi-glowup/src/tools/built-in/bash/display-profile.ts create mode 100644 extensions/pi-glowup/src/tools/built-in/bash/shell-words.ts diff --git a/extensions/pi-glowup/README.md b/extensions/pi-glowup/README.md index 0aa2e467..13977115 100644 --- a/extensions/pi-glowup/README.md +++ b/extensions/pi-glowup/README.md @@ -75,7 +75,9 @@ Global settings are stored in `~/.pi/agent/extension-settings/pi-glowup.json`. | `scriptPreview.showPrologueOmission` | boolean | `false` | Show a count row when collapsed previews omit leading setup imports. | | `scriptPreview.shellLayout` | `preserve` \| `auto` \| `always` | `"auto"` | Choose when composed Bash commands are reflowed at safe syntax boundaries. | | `scriptPreview.shellOperatorPosition` | `trailing` \| `leading` | `"trailing"` | Place Bash chain operators before or after reflowed line breaks. | -| `scriptPreview.formatters` | Record | `{}` | Commands that format script previews through stdin/stdout. | +| `scriptPreview.formatters` | Record | `{}` | Display-only profiles keyed by matching language id; expansion shows original source. | + +### Defaults ```json { @@ -154,6 +156,32 @@ Global settings are stored in `~/.pi/agent/extension-settings/pi-glowup.json`. } } ``` + +### Advanced example + +```json +{ + "$schema": "./schemas/pi-glowup.schema.json", + "scriptPreview": { + "formatters": { + "javascript": { + "kind": "javascript", + "command": [ + "oxfmt" + ], + "objectWrap": "collapse" + }, + "python": { + "kind": "python", + "command": [ + "tuff" + ], + "collections": "ruff-default" + } + } + } +} +``` ## License diff --git a/extensions/pi-glowup/config.schema.json b/extensions/pi-glowup/config.schema.json index cf6efc55..f7432799 100644 --- a/extensions/pi-glowup/config.schema.json +++ b/extensions/pi-glowup/config.schema.json @@ -507,15 +507,269 @@ "type": "object", "patternProperties": { "^.*$": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 + "anyOf": [ + { + "type": "object", + "required": [ + "kind", + "command" + ], + "properties": { + "kind": { + "type": "string", + "const": "command", + "description": "Run custom argv without generated formatter configuration." + }, + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]+$" + }, + "minItems": 1, + "description": "Executable followed by arguments; no shell expansion is performed." + } + }, + "additionalProperties": false, + "title": "CommandFormatter" + }, + { + "type": "object", + "required": [ + "kind", + "command" + ], + "properties": { + "kind": { + "type": "string", + "const": "python", + "description": "Use Tuff's Python formatter." + }, + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]+$" + }, + "minItems": 1, + "description": "Executable followed by arguments; no shell expansion is performed." + }, + "imports": { + "anyOf": [ + { + "type": "string", + "const": "show" + }, + { + "type": "string", + "const": "omit" + } + ], + "description": "Leading import/setup visibility in collapsed display; expansion shows original source.", + "default": "omit" + }, + "indentWidth": { + "type": "integer", + "minimum": 1, + "default": 4, + "description": "Python indentation width." + }, + "collections": { + "anyOf": [ + { + "type": "string", + "const": "fill" + }, + { + "type": "string", + "const": "ruff-default" + } + ], + "default": "ruff-default", + "description": "Collapse collections that fit; fill instead packs items within multiline brackets." + }, + "magicTrailingComma": { + "type": "boolean", + "default": false, + "description": "Treat trailing commas as requests for vertical expansion." + } + }, + "additionalProperties": false, + "title": "PythonFormatter" + }, + { + "type": "object", + "required": [ + "kind", + "command" + ], + "properties": { + "kind": { + "type": "string", + "const": "javascript", + "description": "Use Oxfmt's JavaScript parser." + }, + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]+$" + }, + "minItems": 1, + "description": "Executable followed by arguments; no shell expansion is performed." + }, + "imports": { + "anyOf": [ + { + "type": "string", + "const": "show" + }, + { + "type": "string", + "const": "omit" + } + ], + "description": "Leading import/setup visibility in collapsed display; expansion shows original source.", + "default": "omit" + }, + "tabWidth": { + "type": "integer", + "minimum": 1, + "default": 2, + "description": "Oxfmt indentation width." + }, + "objectWrap": { + "anyOf": [ + { + "type": "string", + "const": "collapse" + }, + { + "type": "string", + "const": "preserve" + } + ], + "default": "collapse", + "description": "Collapse object literals when they fit the actual content width." + } + }, + "additionalProperties": false, + "title": "JavaScriptFormatter" + }, + { + "type": "object", + "required": [ + "kind", + "command" + ], + "properties": { + "kind": { + "type": "string", + "const": "typescript", + "description": "Use Oxfmt's TypeScript parser." + }, + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]+$" + }, + "minItems": 1, + "description": "Executable followed by arguments; no shell expansion is performed." + }, + "imports": { + "anyOf": [ + { + "type": "string", + "const": "show" + }, + { + "type": "string", + "const": "omit" + } + ], + "description": "Leading import/setup visibility in collapsed display; expansion shows original source.", + "default": "omit" + }, + "tabWidth": { + "type": "integer", + "minimum": 1, + "default": 2, + "description": "Oxfmt indentation width." + }, + "objectWrap": { + "anyOf": [ + { + "type": "string", + "const": "collapse" + }, + { + "type": "string", + "const": "preserve" + } + ], + "default": "collapse", + "description": "Collapse object literals when they fit the actual content width." + } + }, + "additionalProperties": false, + "title": "TypeScriptFormatter" + }, + { + "type": "object", + "required": [ + "kind", + "command" + ], + "properties": { + "kind": { + "type": "string", + "const": "bash", + "description": "Use shfmt's Bash formatter without minification." + }, + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\u0000]+$" + }, + "minItems": 1, + "description": "Executable followed by arguments; no shell expansion is performed." + }, + "indentWidth": { + "type": "integer", + "minimum": 1, + "default": 2, + "description": "Bash indentation width." + }, + "caseIndent": { + "type": "boolean", + "default": true, + "description": "Indent Bash case arms." + }, + "spaceRedirects": { + "type": "boolean", + "default": true, + "description": "Separate redirect operators from targets." + }, + "binaryNextLine": { + "type": "boolean", + "default": false, + "description": "Start continued Bash chains with their operator." + } + }, + "additionalProperties": false, + "title": "BashFormatter" + } + ] } }, - "description": "Commands that format script previews through stdin/stdout." + "description": "Display-only profiles keyed by matching language id; expansion shows original source." } }, "additionalProperties": false, diff --git a/extensions/pi-glowup/docs/script-previews.md b/extensions/pi-glowup/docs/script-previews.md index f8ba336d..a8af8b84 100644 --- a/extensions/pi-glowup/docs/script-previews.md +++ b/extensions/pi-glowup/docs/script-previews.md @@ -32,7 +32,7 @@ Reflowed chain and case operators stay at the end of the preceding row by defaul Reflow uses the Bash syntax tree to place logical operators, pipelines, statements, and compound shell bodies on readable rows. It retains the original shell tokens, never extracts language -blocks from a composed command, and leaves submitted multiline commands and heredocs unchanged. +blocks from a composed command, and preserves existing newlines, quoted text, and heredoc bodies. Only clean standalone interpreter calls render as Python, Node, TypeScript, Bun, or Deno previews. Single-quoted inline source inside a composed command remains in the Bash block but receives its language's syntax highlighting when the interpreter can be identified safely. @@ -41,20 +41,45 @@ language's syntax highlighting when the interpreter can be identified safely. Collapsed script previews show up to `scriptPreview.maxCodePreviewLines` lines before adding a hint with Pi's configured tool-expansion key. The default is `8`, and values below `4` are rejected -so four-line-or-shorter previews are never truncated. +while wrapping also counts against the terminal-row budget. -Short scripts keep their imports. Long Python, JavaScript, and TypeScript previews silently omit +Unformatted short scripts keep their imports. Long Python, JavaScript, and TypeScript previews omit only a complete leading import/setup prologue. Set `scriptPreview.showPrologueOmission` to `true` to show the omitted-line count. Expanding the tool call always restores the complete code. ## Formatters -Script previews are not formatted by default. Configure `scriptPreview.formatters` to map rendered language ids to formatter command argv arrays. +Formatting is off by default. `scriptPreview.formatters` maps language ids to explicit profiles: -Formatters apply to clean standalone language calls and codemode JavaScript blocks. Composed Bash -commands are never passed to a language formatter. Successful formatter output is cached by -language, target width, and source for the session. - -`pi-glowup` dynamically adapts formatting to the available terminal rendering width. You can use `{width}`, `{line_length}`, or `{cols}` placeholders in your command arguments (e.g. `["tuff", "format", "--line-length", "{width}", "--stdin-filename", "script.py", "-"]`), and `pi-glowup` will also automatically export environment variables (`TUFF_LINE_LENGTH`, `RUFF_LINE_LENGTH`, `COLUMNS`) containing the available content width for tool rendering. +```json +{ + "javascript": { "kind": "javascript", "command": ["oxfmt"], "objectWrap": "collapse" }, + "typescript": { "kind": "typescript", "command": ["oxfmt"], "tabWidth": 2 }, + "python": { "kind": "python", "command": ["tuff"], "collections": "ruff-default" }, + "bash": { "kind": "bash", "command": ["shfmt"], "indentWidth": 2 } +} +``` -Formatter failures, missing commands, invalid JSON, and empty output are ignored; the original script preview is rendered unchanged. +Python collapses collections that fit, ignores magic trailing commas by default, and retains +comments and literal contents. Explicit `collections: "fill"` keeps multiline brackets and packs rows. JavaScript/TypeScript use Oxfmt object collapse and two-space +indentation. Bash uses two-space shfmt without simplification or minification; only a Bash profile +formats whole shell commands. Executables must already be installed and reachable, or use an +explicit executable path. `kind: "command"` accepts arbitrary formatter argv through `command`. + +Profiles format asynchronously at the available content width and after resize. Oxfmt receives +`printWidth` through an owned temporary configuration; Tuff receives line length and an isolated +collection profile. Blank-line decisions belong to the formatter, never a text-stripping pass. +Python/JS/TS profiles omit complete leading imports by default; `imports: "show"` retains them. +Bash keeps setup statements and heredocs. Expanded Bash-tool and Codemode previews show original source. + +Streaming previews keep raw source until execution completes. Prepared decoration is then +published in one update, or once ready if the result arrives first. Formatting never delays tool +execution or changes submitted arguments. Successful output is cached by language, width, and +source. Command profiles also accept `{width}`, `{line_length}`, and `{cols}` argv placeholders. +Missing or failed executables produce a content-free warning; undecorated source remains visible. + +## Highlighting + +Code within the 40 KiB source budget is tokenized off the TUI thread without a logical-line cutoff. +Plain source remains visible while full grammar-aware colors are pending. Superseded work is cancelled; +the existing cache budgets remain. Tokenizer CPU time is not bounded by an elapsed-time limit. diff --git a/extensions/pi-glowup/scripts/build.mjs b/extensions/pi-glowup/scripts/build.mjs index 6c0bff9e..4c1ad933 100644 --- a/extensions/pi-glowup/scripts/build.mjs +++ b/extensions/pi-glowup/scripts/build.mjs @@ -42,7 +42,10 @@ const result = await esbuild.build({ platform: "node", format: "esm", target: "node22.19", - define: { "import.meta.glowupThemeDirectory": JSON.stringify("../../themes") }, + define: { + "import.meta.glowupThemeDirectory": JSON.stringify("../../themes"), + "import.meta.glowupSyntaxWorker": JSON.stringify("./tokenize-worker.mjs"), + }, treeShaking: true, sourcemap: true, sourcesContent: true, @@ -67,12 +70,31 @@ const result = await esbuild.build({ metafile: true, }); +const workerResult = await esbuild.build({ + absWorkingDir: packageRoot, + entryPoints: ["src/rendering/syntax/tokenize-worker.ts"], + outfile: path.join(outputRoot, "src", "tokenize-worker.mjs"), + bundle: true, + platform: "node", + format: "esm", + target: "node22.19", + treeShaking: true, + sourcemap: true, + sourcesContent: true, + legalComments: "none", + external, + metafile: true, +}); + await copyFile( path.join(packageRoot, "config.schema.json"), path.join(outputRoot, "config.schema.json"), ); -const bundledHostInputs = Object.keys(result.metafile.inputs).filter((input) => +const bundledHostInputs = [ + ...Object.keys(result.metafile.inputs), + ...Object.keys(workerResult.metafile.inputs), +].filter((input) => /node_modules\/(?:@earendil-works|@mariozechner|@sinclair\/typebox|shiki|typebox)\//u.test(input), ); if (bundledHostInputs.length > 0) { diff --git a/extensions/pi-glowup/src/pi/lifecycle.ts b/extensions/pi-glowup/src/pi/lifecycle.ts index 1af551a3..8b4fba6e 100644 --- a/extensions/pi-glowup/src/pi/lifecycle.ts +++ b/extensions/pi-glowup/src/pi/lifecycle.ts @@ -137,7 +137,7 @@ export function installGlowup(pi: Pick): void { const edit = createNativeEditFeature(); const bash = createNativeBashFeature(); - const codemode = createCodemodePreview(); + const codemode = createCodemodePreview(() => formatter); const deletion = createNativeDeleteFeature(); const exploration = createExplorationFeature(); const { captureNativeEditSnapshot, finishNativeEditSnapshot } = edit; @@ -158,6 +158,7 @@ export function installGlowup(pi: Pick): void { let restoreAbort: AbortController | undefined; const clearSessionState = (): void => { + syntax.cancel(); restoreAbort?.abort(); restoreAbort = undefined; edit.clear(); @@ -381,6 +382,7 @@ export function installGlowup(pi: Pick): void { toolCallId: event.toolCallId, code, formatter, + completed: true, isCurrent: () => sessionGeneration === formatterGeneration, invalidate: () => refreshToolRows(ctx), }); diff --git a/extensions/pi-glowup/src/pi/patches/markdown-syntax.ts b/extensions/pi-glowup/src/pi/patches/markdown-syntax.ts index 3674ed87..d00a0ac4 100644 --- a/extensions/pi-glowup/src/pi/patches/markdown-syntax.ts +++ b/extensions/pi-glowup/src/pi/patches/markdown-syntax.ts @@ -1,6 +1,11 @@ import { Markdown, type MarkdownTheme } from "@earendil-works/pi-tui"; -import { highlightSyntaxCode } from "../../rendering/syntax/highlighter.ts"; +import { + highlightSyntaxCode, + observeSyntaxHighlightSource, + syntaxHighlightingVersion, +} from "../../rendering/syntax/highlighter.ts"; import { normalizedCodeLines } from "../../text-boundaries.ts"; +import { scheduleCodeOutputSyntaxLoad } from "../../rendering/syntax/code-component.ts"; const MARKDOWN_PATCH_KEY = Symbol.for("zigai.pi-glowup.syntax-markdown"); const MARKDOWN_PATCH_STATE_KEY = Symbol.for("zigai.pi-glowup.syntax-markdown.state"); @@ -15,6 +20,8 @@ let markdownThinkingThemeSuppressionFailures = 0; type MarkdownInstance = { readonly theme?: MarkdownTheme; + readonly text?: string; + invalidate?: () => void; readonly defaultTextStyle?: { readonly italic?: boolean }; }; @@ -42,8 +49,24 @@ export type MarkdownSyntaxPatchStats = { readonly thinkingThemeSuppressionFailures: number; }; +const markdownOwners = new WeakMap< + MarkdownInstance, + { readonly invalidate: () => void; version: number } +>(); +let activeMarkdownInvalidation: (() => void) | undefined; +let activeMarkdownSource: (() => boolean) | undefined; + function highlightMarkdownCode(code: string, lang?: string): string[] { - return markdownSyntaxEnabled ? highlightSyntaxCode(code, lang) : normalizedCodeLines(code); + if (!markdownSyntaxEnabled) return normalizedCodeLines(code); + + scheduleCodeOutputSyntaxLoad( + { ...(lang !== undefined && { language: lang }) }, + activeMarkdownInvalidation, + code, + activeMarkdownSource, + ); + + return highlightSyntaxCode(code, lang, { invalidate: activeMarkdownInvalidation }); } /** Returns Markdown syntax patch counters for debug diagnostics. */ @@ -96,6 +119,32 @@ export function configureMarkdownSyntaxPatch( const originalRender = prototype.render; const wrapperRender = function renderWithGlowupSyntax(this: MarkdownInstance, width: number) { + let owner = markdownOwners.get(this); + if (owner === undefined) { + owner = { + invalidate: () => { + this.invalidate?.(); + }, + version: syntaxHighlightingVersion(), + }; + markdownOwners.set(this, owner); + } + + const version = syntaxHighlightingVersion(); + if (owner.version !== version) { + owner.version = version; + owner.invalidate(); + } + + let current: (() => boolean) | undefined; + if (this.text !== undefined) + current = observeSyntaxHighlightSource(this.text, owner.invalidate); + + const previousInvalidation = activeMarkdownInvalidation; + const previousSource = activeMarkdownSource; + activeMarkdownInvalidation = owner.invalidate; + activeMarkdownSource = current; + const restoreSyntaxTheme = prototype[MARKDOWN_PATCH_STATE_KEY]?.enabled !== false ? prepareSyntaxTheme(this) @@ -104,6 +153,8 @@ export function configureMarkdownSyntaxPatch( try { return originalRender?.call(this, width) ?? []; } finally { + activeMarkdownInvalidation = previousInvalidation; + activeMarkdownSource = previousSource; restoreSyntaxTheme?.(); } }; diff --git a/extensions/pi-glowup/src/pi/syntax-lifecycle.ts b/extensions/pi-glowup/src/pi/syntax-lifecycle.ts index 83f5f26f..35ccfc65 100644 --- a/extensions/pi-glowup/src/pi/syntax-lifecycle.ts +++ b/extensions/pi-glowup/src/pi/syntax-lifecycle.ts @@ -1,7 +1,11 @@ import { type ExtensionContext } from "@earendil-works/pi-coding-agent"; import { type GlowupConfig } from "../config/normalize.ts"; import { DebugFileLogger, type DebugLogFields } from "../diagnostics/logger.ts"; -import { refreshSyntaxHighlighting } from "../rendering/syntax/highlighter.ts"; +import { + clearSyntaxHighlightCache, + onSyntaxHighlightComplete, + refreshSyntaxHighlighting, +} from "../rendering/syntax/highlighter.ts"; import { refreshToolRows } from "./session.ts"; const SYNTAX_WARMUP_DELAY_MS = 5_000; @@ -52,6 +56,7 @@ export function createSyntaxLifecycle(options: { } | undefined; let syntaxTimer: NodeJS.Timeout | undefined; + let stopCompletionNotifications: (() => void) | undefined; const startPendingSyntaxHighlighting = (): void => { const pending = pendingSyntaxStart; @@ -89,6 +94,13 @@ export function createSyntaxLifecycle(options: { context: ExtensionContext, ): void => { if (syntaxTimer !== undefined) clearTimeout(syntaxTimer); + stopCompletionNotifications?.(); + + const currentGeneration = generation(); + + stopCompletionNotifications = onSyntaxHighlightComplete(() => { + if (generation() === currentGeneration) refreshToolRows(context); + }); pendingSyntaxStart = { generation: generation(), @@ -100,6 +112,9 @@ export function createSyntaxLifecycle(options: { }; function cancel(): void { + stopCompletionNotifications?.(); + stopCompletionNotifications = undefined; + clearSyntaxHighlightCache(); pendingSyntaxStart = undefined; if (syntaxTimer !== undefined) { diff --git a/extensions/pi-glowup/src/rendering/diff/text-renderer.ts b/extensions/pi-glowup/src/rendering/diff/text-renderer.ts index 0d2bad44..9604cfb2 100644 --- a/extensions/pi-glowup/src/rendering/diff/text-renderer.ts +++ b/extensions/pi-glowup/src/rendering/diff/text-renderer.ts @@ -22,7 +22,8 @@ import { wrapStyledText, makeComponent, toolExpandHint, wrapPrefixedLine } from import { truncateToWidth, visibleWidth, type Component } from "@earendil-works/pi-tui"; import { expandTerminalTabs } from "../../text-boundaries.ts"; import { strongerDiffBackgroundAnsi } from "./ansi-colors.ts"; -import { highlightCodeOutput } from "../syntax/code-component.ts"; +import { highlightCodeOutput, scheduleCodeOutputSyntaxLoad } from "../syntax/code-component.ts"; +import { observeSyntaxHighlightSource, syntaxHighlightingVersion } from "../syntax/highlighter.ts"; import { selectSemanticDiffIndices, type SemanticDiffRowKind, @@ -366,6 +367,8 @@ function styleDiffContent( function highlightDiffContents( lines: ReadonlyArray, filePath: string | undefined, + invalidate: () => void, + isCurrent: () => boolean, ): ReadonlyArray { if (filePath === undefined) { return []; @@ -380,9 +383,10 @@ function highlightDiffContents( return; } - const highlightedLines = highlightCodeOutput(run.map((row) => row.content).join("\n"), { - path: syntaxPath, - }); + const content = run.map((row) => row.content).join("\n"); + scheduleCodeOutputSyntaxLoad({ path: syntaxPath }, invalidate, content, isCurrent); + + const highlightedLines = highlightCodeOutput(content, { path: syntaxPath, invalidate }); if (highlightedLines.length === run.length) { for (const [rowIndex, row] of run.entries()) { const highlighted = highlightedLines[rowIndex]; @@ -476,7 +480,13 @@ export function renderGlowupDiff( expanded: boolean, options: GlowupDiffRenderOptions = {}, ): Component { - return makeComponent((width) => { + let isCurrent = (): boolean => false; + let syntaxVersion: number | undefined; + const invalidate = (): void => { + if (isCurrent()) component.invalidate(); + }; + + const component = makeComponent((width) => { const allDiffLineCount = sections.reduce( (count, section) => count + section.lines.length, 0, @@ -547,7 +557,12 @@ export function renderGlowupDiff( section.lines, section.lineCoordinates, ); - const highlightedContents = highlightDiffContents(section.lines, section.path); + const highlightedContents = highlightDiffContents( + section.lines, + section.path, + invalidate, + isCurrent, + ); const changedRanges = changedRangesForDiffLines(section.lines); const renderLines = ( lines: ReadonlyArray<{ readonly line: string; readonly index: number }>, @@ -596,4 +611,20 @@ export function renderGlowupDiff( return rendered; }); + + return { + render(width) { + const version = syntaxHighlightingVersion(); + if (syntaxVersion !== version) { + syntaxVersion = version; + isCurrent = observeSyntaxHighlightSource(sections, invalidate); + component.invalidate(); + } + + return component.render(width); + }, + invalidate() { + component.invalidate(); + }, + }; } diff --git a/extensions/pi-glowup/src/rendering/syntax/code-component.ts b/extensions/pi-glowup/src/rendering/syntax/code-component.ts index 8f8e78e0..697bb17b 100644 --- a/extensions/pi-glowup/src/rendering/syntax/code-component.ts +++ b/extensions/pi-glowup/src/rendering/syntax/code-component.ts @@ -2,6 +2,8 @@ import { normalizeSyntaxLanguage, syntaxLanguageFromFile } from "./language.ts"; import { getLoadedSyntaxHighlighterForLanguage, highlightSyntaxCode, + highlightSyntaxCodeAsync, + observeSyntaxHighlightSource, loadSyntaxLanguageIfReady, onSyntaxHighlightingStateChange, } from "./highlighter.ts"; @@ -10,11 +12,12 @@ export type CodeOutputSyntax = { readonly language?: string; readonly path?: string; readonly cache?: boolean | undefined; + readonly invalidate?: (() => void) | undefined; }; const MAX_STRUCTURED_OUTPUT_DETECTION_CHARS = 64 * 1024; const pendingCodeOutputSyntaxLoads = new Map void>>(); -const activeCodeOutputSyntaxLoads = new Set(); +const activeCodeOutputSyntaxLoads = new Map>(); onSyntaxHighlightingStateChange((status) => { if (status === "ready") { @@ -25,16 +28,17 @@ onSyntaxHighlightingStateChange((status) => { return; } - if (status === "disabled" || status === "failed") { - pendingCodeOutputSyntaxLoads.clear(); - activeCodeOutputSyntaxLoads.clear(); - } + pendingCodeOutputSyntaxLoads.clear(); + activeCodeOutputSyntaxLoads.clear(); }); /** Highlights code-like output when a language or path is known; otherwise returns normalized plain lines. */ export function highlightCodeOutput(text: string, syntax: CodeOutputSyntax | undefined): string[] { const language = syntax?.language ?? syntaxLanguageFromFile(syntax?.path, text); - return highlightSyntaxCode(text, language, { cache: syntax?.cache }); + return highlightSyntaxCode(text, language, { + cache: syntax?.cache, + invalidate: syntax?.invalidate, + }); } /** Schedules async loading for path/language hints so later sync renders can highlight them. */ @@ -42,29 +46,48 @@ export function scheduleCodeOutputSyntaxLoad( syntax: CodeOutputSyntax | undefined, invalidate: (() => void) | undefined, content?: string, + current?: () => boolean, ): void { if (invalidate === undefined) { return; } + let isCurrent = current; + if (isCurrent === undefined && content !== undefined) + isCurrent = observeSyntaxHighlightSource(content, invalidate); + const language = syntax?.language ?? syntaxLanguageFromFile(syntax?.path, content); const normalizedLanguage = normalizeSyntaxLanguage(language); if (normalizedLanguage === undefined || normalizedLanguage === "text") { return; } + const refresh = (): void => { + if (isCurrent !== undefined && !isCurrent()) return; + + if (content === undefined) { + invalidate(); + return; + } + + void highlightSyntaxCodeAsync(content, normalizedLanguage, { + invalidate, + cache: syntax?.cache, + }); + }; if (getLoadedSyntaxHighlighterForLanguage(normalizedLanguage) !== undefined) { + if (content !== undefined) refresh(); return; } const pendingInvalidations = pendingCodeOutputSyntaxLoads.get(normalizedLanguage); if (pendingInvalidations !== undefined) { - pendingInvalidations.add(invalidate); + pendingInvalidations.add(refresh); requestPendingCodeOutputSyntaxLoad(normalizedLanguage); return; } - const invalidations = new Set([invalidate]); + const invalidations = new Set([refresh]); pendingCodeOutputSyntaxLoads.set(normalizedLanguage, invalidations); requestPendingCodeOutputSyntaxLoad(normalizedLanguage); } @@ -90,10 +113,11 @@ function requestPendingCodeOutputSyntaxLoad(language: string): void { return; } - activeCodeOutputSyntaxLoads.add(language); - void loadSyntaxLanguageIfReady(language) + const loading = loadSyntaxLanguageIfReady(language); + activeCodeOutputSyntaxLoads.set(language, loading); + void loading .then((loaded) => { - if (loaded) { + if (loaded && pendingCodeOutputSyntaxLoads.get(language) === invalidations) { pendingCodeOutputSyntaxLoads.delete(language); for (const invalidatePendingPreview of invalidations) { @@ -102,10 +126,12 @@ function requestPendingCodeOutputSyntaxLoad(language: string): void { } }) .catch(() => { - pendingCodeOutputSyntaxLoads.delete(language); + if (pendingCodeOutputSyntaxLoads.get(language) === invalidations) + pendingCodeOutputSyntaxLoads.delete(language); }) .finally(() => { - activeCodeOutputSyntaxLoads.delete(language); + if (activeCodeOutputSyntaxLoads.get(language) === loading) + activeCodeOutputSyntaxLoads.delete(language); }); } @@ -159,7 +185,7 @@ function looksLikeXml(text: string): boolean { function looksLikeShellSnippet(text: string): boolean { const firstLine = text.split("\n", 1)[0] ?? ""; - return /^\s*(?:[$#]\s+)?(?:npm|pnpm|yarn|bun|node|python|python3|git|rg|grep|find|sed|awk|curl|mkdir|rm|cp|mv|cat|ls|cd)\b/u.test( + return /^\s*(?:[$#]\s+)?(?:npm|pnpm|yarn|bun|node|python|python3|git|rg|grep|find|sed|awk|curl|mkdir|rm|cp|mv|cat|ls|cd)(?=$|[\t ]+(?![\t ]|ERR!(?:\s|$)|error\b))/u.test( firstLine, ); } diff --git a/extensions/pi-glowup/src/rendering/syntax/highlighter.ts b/extensions/pi-glowup/src/rendering/syntax/highlighter.ts index bea90edb..6aaff193 100644 --- a/extensions/pi-glowup/src/rendering/syntax/highlighter.ts +++ b/extensions/pi-glowup/src/rendering/syntax/highlighter.ts @@ -2,7 +2,8 @@ import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; import type { BundledLanguage, CreateHighlighterFactory, HighlighterGeneric } from "shiki"; import { normalizedCodeLines } from "../../text-boundaries.ts"; -import { tokensToAnsiLines } from "./ansi.ts"; +import type { DiffSection } from "../diff/text-diff.ts"; +import { createSyntaxTokenizer } from "./tokenizer.ts"; import { configureBracketPairColoring } from "./brackets.ts"; import { isBundledSyntaxLanguage, @@ -23,12 +24,12 @@ import { const MAX_CODE_BYTES = 40 * 1024; const MAX_CACHEABLE_CODE_BYTES = 8 * 1024; const MAX_CACHE_BYTES = 512 * 1024; -const MAX_LINE_LENGTH = 2_000; -const TOKENIZE_MAX_LINE_LENGTH = MAX_LINE_LENGTH; const CACHE_LIMIT = 100; export type SyntaxHighlightOptions = { readonly cache?: boolean | undefined; + readonly invalidate?: (() => void) | undefined; + readonly signal?: AbortSignal; }; export type SyntaxHighlightCacheStats = { @@ -97,6 +98,7 @@ type SyntaxState = readonly config: SyntaxConfig; readonly theme: LoadedSyntaxTheme; readonly highlighter: HighlighterGeneric; + readonly tokenizer: ReturnType; readonly loadedLanguages: Set; readonly dynamicLanguages: Set; }; @@ -110,6 +112,22 @@ const highlightedCodeCache = new Map< { readonly lines: string[]; readonly bytes: number } >(); +type HighlightOwner = { + readonly source: string | readonly DiffSection[]; + readonly controller: AbortController; + readonly results: Map; + readonly pending: Map>; +}; + +let bracketPairColoringEnabled = true; +let highlightOwners = new WeakMap<() => void, HighlightOwner>(); +const pendingHighlights = new Map>(); +const highlightControllers = new Set(); +const tokenizerDisposals = new Set>(); +const highlightCompletionListeners = new Set<() => void>(); +const activeHighlights = new Set>(); +const completedUnownedHighlights = new Map>(); + let highlightedCodeCacheBytes = 0; const syntaxStateListeners = new Set(); @@ -253,6 +271,7 @@ export function configureSyntaxBracketPairColoring(enabled: boolean): void { return; } + bracketPairColoringEnabled = enabled; clearSyntaxHighlightCache(); syntaxRenderingVersion += 1; } @@ -266,7 +285,19 @@ export function onSyntaxHighlightingStateChange(listener: SyntaxStateListener): }; } -/** Highlights code synchronously for render-time callers, falling back to plain lines when unavailable. */ +export function onSyntaxHighlightComplete(listener: () => void): () => void { + highlightCompletionListeners.add(listener); + + return () => { + highlightCompletionListeners.delete(listener); + }; +} + +export async function settleSyntaxHighlights(): Promise { + while (activeHighlights.size > 0) await Promise.allSettled(activeHighlights); +} + +/** Returns completed worker colors or plain lines while owned tokenization is pending. */ export function highlightSyntaxCode( code: string, language: string | undefined, @@ -288,40 +319,159 @@ export function highlightSyntaxCode( return normalizedCodeLines(code); } - const shouldCache = options.cache !== false && codeByteLength <= MAX_CACHEABLE_CODE_BYTES; - const cacheKey = shouldCache - ? highlightedCodeCacheKey(state.theme.name, normalizedLanguage, code, codeByteLength) - : undefined; - if (cacheKey !== undefined) { - const cached = highlightedCodeCache.get(cacheKey); - if (cached) { - return cached.lines; - } + const key = highlightedCodeCacheKey(state.theme.name, normalizedLanguage, code, codeByteLength); + const invalidate = options.invalidate; + if (invalidate !== undefined && !highlightOwners.has(invalidate)) + observeSyntaxHighlightSource(code, invalidate); + + const owned = invalidate === undefined ? undefined : highlightOwners.get(invalidate); + const completed = owned?.results.get(key); + if (completed !== undefined) return completed; + + const cached = options.cache === false ? undefined : highlightedCodeCache.get(key); + if (cached !== undefined) { + owned?.results.set(key, cached.lines); + return cached.lines; } - const plainLines = normalizedCodeLines(code); - if (plainLines.some((line) => line.length > MAX_LINE_LENGTH)) { - return plainLines; + const unowned = completedUnownedHighlights.get(key)?.deref(); + if (unowned !== undefined && options.cache !== false) { + owned?.results.set(key, unowned); + return unowned; } - try { - const tokenRows = state.highlighter.codeToTokensBase(code, { - lang: normalizedLanguage, - theme: state.theme.registration, - tokenizeMaxLineLength: TOKENIZE_MAX_LINE_LENGTH, - }); - const highlighted = normalizeHighlightedLineCount( - tokensToAnsiLines(tokenRows, normalizedLanguage), - plainLines.length, - ); - if (cacheKey !== undefined) { - rememberHighlightedCode(cacheKey, highlighted); - } + void highlightSyntaxCodeAsync(code, normalizedLanguage, options); + + return normalizedCodeLines(code); +} + +/** Cancels obsolete work when a renderer observes a different submitted source. */ +export function observeSyntaxHighlightSource( + source: string | readonly DiffSection[], + invalidate: () => void, +): () => boolean { + const previous = highlightOwners.get(invalidate); + if (previous?.source === source) return () => highlightOwners.get(invalidate) === previous; + + previous?.controller.abort(); + previous?.results.clear(); + + const owner: HighlightOwner = { + source, + controller: new AbortController(), + results: new Map(), + pending: new Map(), + }; + highlightOwners.set(invalidate, owner); - return highlighted; - } catch { + return () => highlightOwners.get(invalidate) === owner; +} + +/** Computes complete code colors off the TUI thread, preserving grammar state across all lines. */ +export async function highlightSyntaxCodeAsync( + code: string, + language: string | undefined, + options: SyntaxHighlightOptions = {}, +): Promise { + const state = syntaxState; + const normalizedLanguage = normalizeSyntaxLanguage(language); + const plainLines = normalizedCodeLines(code); + const size = Buffer.byteLength(code, "utf8"); + + if ( + state?.status !== "ready" || + normalizedLanguage === undefined || + normalizedLanguage === "text" || + !state.loadedLanguages.has(normalizedLanguage) || + size > MAX_CODE_BYTES || + options.signal?.aborted === true + ) return plainLines; - } + + const key = highlightedCodeCacheKey(state.theme.name, normalizedLanguage, code, size); + const shouldCache = options.cache !== false && size <= MAX_CACHEABLE_CODE_BYTES; + const invalidate = options.invalidate; + if (invalidate !== undefined && !highlightOwners.has(invalidate)) + observeSyntaxHighlightSource(code, invalidate); + + const owner = invalidate === undefined ? undefined : highlightOwners.get(invalidate); + const ownedResult = owner?.results.get(key); + if (ownedResult !== undefined) return ownedResult; + + const cached = shouldCache ? highlightedCodeCache.get(key) : undefined; + if (cached !== undefined) { + owner?.results.set(key, cached.lines); + return cached.lines; + } + + const generation = syntaxRenderingVersion; + const controller = new AbortController(); + const signals = [controller.signal]; + if (owner !== undefined) signals.push(owner.controller.signal); + if (options.signal !== undefined) signals.push(options.signal); + + const signal = AbortSignal.any(signals); + const pendingKey = owner === undefined && options.signal === undefined ? key : undefined; + let pending = + owner?.pending.get(key) ?? + (pendingKey === undefined ? undefined : pendingHighlights.get(pendingKey)); + if (pending === undefined) { + highlightControllers.add(controller); + pending = state.tokenizer + .tokenize(plainLines.join("\n"), normalizedLanguage, bracketPairColoringEnabled, { + signal, + }) + .then((lines) => { + if ( + signal.aborted || + syntaxState !== state || + syntaxRenderingVersion !== generation || + lines === undefined + ) + return plainLines; + + const highlighted = normalizeHighlightedLineCount(lines, plainLines.length); + if (shouldCache) rememberHighlightedCode(key, highlighted); + + if ( + owner !== undefined && + invalidate !== undefined && + highlightOwners.get(invalidate) === owner + ) { + owner.results.set(key, highlighted); + invalidate(); + } else if (!shouldCache && options.cache !== false) { + if (completedUnownedHighlights.size >= CACHE_LIMIT) { + const oldest = completedUnownedHighlights.keys().next().value; + if (oldest !== undefined) completedUnownedHighlights.delete(oldest); + } + + completedUnownedHighlights.set(key, new WeakRef(highlighted)); + } + + for (const listener of highlightCompletionListeners) listener(); + + return highlighted; + }) + .catch(() => plainLines) + .finally(() => { + highlightControllers.delete(controller); + + if (pending !== undefined) activeHighlights.delete(pending); + + if (pendingKey !== undefined && pendingHighlights.get(pendingKey) === pending) + pendingHighlights.delete(pendingKey); + + if (owner !== undefined && owner.pending.get(key) === pending) + owner.pending.delete(key); + }); + activeHighlights.add(pending); + + if (pendingKey !== undefined) pendingHighlights.set(pendingKey, pending); + owner?.pending.set(key, pending); + } + + return pending; } export type LoadedSyntaxHighlighter = { @@ -437,6 +587,13 @@ export async function loadSyntaxLanguageIfReady(language: string | undefined): P } export function clearSyntaxHighlightCache(): void { + syntaxRenderingVersion += 1; + + for (const controller of highlightControllers) controller.abort(); + highlightControllers.clear(); + pendingHighlights.clear(); + highlightOwners = new WeakMap(); + completedUnownedHighlights.clear(); highlightedCodeCache.clear(); highlightedCodeCacheBytes = 0; } @@ -510,6 +667,7 @@ export async function disposeSyntaxHighlighting(): Promise { activeConfigurationKey = undefined; disposeReadySyntaxState(state); + await Promise.allSettled(tokenizerDisposals); syntaxRenderingVersion += 1; notifySyntaxStateListeners("uninitialized"); } @@ -565,6 +723,12 @@ function notifySyntaxStateListeners(status: SyntaxStateStatus): void { function disposeReadySyntaxState(state: SyntaxState | undefined): void { if (state?.status === "ready") { state.highlighter.dispose(); + + const disposal = state.tokenizer.dispose(); + tokenizerDisposals.add(disposal); + void disposal.finally(() => { + tokenizerDisposals.delete(disposal); + }); } } @@ -600,11 +764,20 @@ async function initializeSyntaxHighlightingOnce( themes: [theme.registration], langs: [...preloadSelection.languages], }); + return { status: "ready", config, theme, highlighter, + tokenizer: createSyntaxTokenizer(theme.registration, { + maxJobs: CACHE_LIMIT, + maxBytes: MAX_CACHE_BYTES, + reportWarning: options.reportWarning, + async waitForDisposals() { + await Promise.all(tokenizerDisposals); + }, + }), loadedLanguages: new Set(highlighter.getLoadedLanguages()), dynamicLanguages: new Set(), }; diff --git a/extensions/pi-glowup/src/rendering/syntax/tokenize-worker.ts b/extensions/pi-glowup/src/rendering/syntax/tokenize-worker.ts new file mode 100644 index 00000000..6cc57705 --- /dev/null +++ b/extensions/pi-glowup/src/rendering/syntax/tokenize-worker.ts @@ -0,0 +1,53 @@ +import { parentPort } from "node:worker_threads"; +import type { BundledLanguage, ThemeRegistration } from "shiki"; +import { tokensToAnsiLines } from "./ansi.ts"; +import { configureBracketPairColoring } from "./brackets.ts"; + +export type TokenizeRequest = { + readonly id: number; + readonly code: string; + readonly language: BundledLanguage; + readonly theme: ThemeRegistration; + readonly brackets: boolean; +}; + +export type TokenizeResponse = + | { readonly id: number; readonly lines: string[] } + | { readonly id: number; readonly error: string }; + +const port = parentPort; +if (port === null) throw new Error("Syntax worker requires a parent port"); + +const { createHighlighter } = await import("shiki"); +let highlighter: Awaited> | undefined; + +async function tokenize(request: TokenizeRequest): Promise { + try { + highlighter ??= await createHighlighter({ themes: [request.theme], langs: [] }); + await highlighter.loadLanguage(request.language); + configureBracketPairColoring(request.brackets); + + const tokens = highlighter.codeToTokensBase(request.code, { + lang: request.language, + theme: request.theme, + tokenizeMaxLineLength: 0, + tokenizeTimeLimit: 0, + }); + return { id: request.id, lines: tokensToAnsiLines(tokens, request.language) }; + } catch (cause: unknown) { + return { + id: request.id, + error: cause instanceof Error ? cause.message : String(cause), + }; + } +} + +port.on("message", (request: TokenizeRequest) => { + void tokenize(request) + .then((response) => { + port.postMessage(response); + }) + .catch(() => { + port.close(); + }); +}); diff --git a/extensions/pi-glowup/src/rendering/syntax/tokenizer.ts b/extensions/pi-glowup/src/rendering/syntax/tokenizer.ts new file mode 100644 index 00000000..f51907f7 --- /dev/null +++ b/extensions/pi-glowup/src/rendering/syntax/tokenizer.ts @@ -0,0 +1,220 @@ +import { Worker } from "node:worker_threads"; +import type { BundledLanguage, ThemeRegistration } from "shiki"; +import type { TokenizeRequest, TokenizeResponse } from "./tokenize-worker.ts"; + +declare global { + interface ImportMeta { + readonly glowupSyntaxWorker?: string; + } +} + +type Work = { + readonly request: TokenizeRequest; + readonly bytes: number; + readonly resolve: (lines: string[] | undefined) => void; + readonly signal: AbortSignal | undefined; + readonly cancel: () => void; +}; + +export function createSyntaxTokenizer( + theme: ThemeRegistration, + options: { + readonly maxJobs: number; + readonly maxBytes: number; + readonly reportWarning: ((message: string) => void) | undefined; + readonly waitForDisposals: () => Promise; + }, +) { + const queue: Work[] = []; + const terminations = new Set>(); + let active: Work | undefined; + let worker: Worker | undefined; + let starting: Promise | undefined; + let nextId = 0; + let bytes = 0; + let disposed = false; + let failed = false; + + function finish(work: Work, lines: string[] | undefined): void { + bytes -= work.bytes; + work.signal?.removeEventListener("abort", work.cancel); + work.resolve(lines); + } + + function stopWorker(): void { + const previous = worker; + worker = undefined; + + if (previous === undefined) return; + + const termination = previous.terminate(); + terminations.add(termination); + void termination + .then(() => { + terminations.delete(termination); + dispatch(); + }) + .catch(() => { + terminations.delete(termination); + fail(); + }); + } + + function fail(): void { + if (failed || disposed) return; + + failed = true; + + if (active !== undefined) finish(active, undefined); + active = undefined; + + for (const work of queue.splice(0)) finish(work, undefined); + stopWorker(); + options.reportWarning?.("[pi-glowup] Syntax worker failed; reload to retry highlighting"); + } + + async function startWorker(): Promise { + try { + await options.waitForDisposals(); + + if (disposed || failed || queue.length === 0) return; + + const current = new Worker( + new URL(import.meta.glowupSyntaxWorker ?? "./tokenize-worker.ts", import.meta.url), + { execArgv: [] }, + ); + worker = current; + current.on("error", () => { + if (worker === current) fail(); + }); + + current.on("exit", () => { + if (worker === current) fail(); + }); + + current.on("message", (response: TokenizeResponse) => { + if (worker !== current || active?.request.id !== response.id) return; + + const completed = active; + active = undefined; + + if ("error" in response) { + finish(completed, undefined); + fail(); + } else { + finish(completed, response.lines); + dispatch(); + } + }); + } catch { + fail(); + } + } + + function dispatch(): void { + if ( + disposed || + failed || + active !== undefined || + terminations.size > 0 || + starting !== undefined + ) + return; + + if (queue.length === 0) { + worker?.unref(); + return; + } + + if (worker === undefined) { + const start = startWorker(); + starting = start; + void start.then( + () => { + if (starting === start) starting = undefined; + dispatch(); + }, + () => { + if (starting === start) starting = undefined; + fail(); + }, + ); + + return; + } + + const work = queue.shift(); + if (work === undefined) return; + + active = work; + + try { + worker.ref(); + worker.postMessage(work.request); + } catch { + fail(); + } + } + + async function tokenize( + code: string, + language: BundledLanguage, + brackets: boolean, + cancellation: { readonly signal?: AbortSignal } = {}, + ): Promise { + const size = Buffer.byteLength(code, "utf8"); + if ( + disposed || + failed || + cancellation.signal?.aborted === true || + queue.length + Number(active !== undefined) >= options.maxJobs || + bytes + size > options.maxBytes + ) + return undefined; + + return new Promise((resolve) => { + const work: Work = { + request: { id: nextId++, code, language, theme, brackets }, + bytes: size, + resolve, + signal: cancellation.signal, + cancel: () => { + if (active === work) { + active = undefined; + finish(work, undefined); + stopWorker(); + dispatch(); + return; + } + + const index = queue.indexOf(work); + if (index >= 0) { + queue.splice(index, 1); + finish(work, undefined); + } + }, + }; + + bytes += size; + cancellation.signal?.addEventListener("abort", work.cancel, { once: true }); + queue.push(work); + dispatch(); + }); + } + + async function dispose(): Promise { + disposed = true; + + if (active !== undefined) finish(active, undefined); + active = undefined; + + for (const work of queue.splice(0)) finish(work, undefined); + stopWorker(); + + const closing: Array> = [...terminations]; + if (starting !== undefined) closing.push(starting); + await Promise.allSettled(closing); + } + + return { tokenize, dispose }; +} diff --git a/extensions/pi-glowup/src/settings-input.ts b/extensions/pi-glowup/src/settings-input.ts index e481d443..1dae1aad 100644 --- a/extensions/pi-glowup/src/settings-input.ts +++ b/extensions/pi-glowup/src/settings-input.ts @@ -1,3 +1,4 @@ +import type { ExtensionSettingsDefinitionInput } from "@zigai/pi-extension-settings"; import Type, { type StaticDecode } from "typebox"; import { DEFAULT_MUTATION_SETTINGS } from "./rendering/preview-settings.ts"; @@ -292,10 +293,112 @@ const patchesSchema = Type.Object( }, ); -const formatterCommandSchema = Type.Array(Type.String({ minLength: 1 }), { +const formatterCommandSchema = Type.Array(Type.String({ minLength: 1, pattern: "^[^\\u0000]+$" }), { minItems: 1, + description: "Executable followed by arguments; no shell expansion is performed.", }); +const formatterImportsSchema = Type.Optional( + Type.Union([Type.Literal("show"), Type.Literal("omit")], { + description: + "Leading import/setup visibility in collapsed display; expansion shows original source.", + default: "omit", + }), +); + +const javascriptFormatterProperties = { + command: formatterCommandSchema, + imports: formatterImportsSchema, + tabWidth: Type.Optional( + Type.Integer({ minimum: 1, default: 2, description: "Oxfmt indentation width." }), + ), + objectWrap: Type.Optional( + Type.Union([Type.Literal("collapse"), Type.Literal("preserve")], { + default: "collapse", + description: "Collapse object literals when they fit the actual content width.", + }), + ), +}; + +export const scriptFormatterProfileSchema = Type.Union([ + Type.Object( + { + kind: Type.Literal("command", { + description: "Run custom argv without generated formatter configuration.", + }), + command: formatterCommandSchema, + }, + { additionalProperties: false, title: "CommandFormatter" }, + ), + Type.Object( + { + kind: Type.Literal("python", { description: "Use Tuff's Python formatter." }), + command: formatterCommandSchema, + imports: formatterImportsSchema, + indentWidth: Type.Optional( + Type.Integer({ minimum: 1, default: 4, description: "Python indentation width." }), + ), + collections: Type.Optional( + Type.Union([Type.Literal("fill"), Type.Literal("ruff-default")], { + default: "ruff-default", + description: + "Collapse collections that fit; fill instead packs items within multiline brackets.", + }), + ), + magicTrailingComma: Type.Optional( + Type.Boolean({ + default: false, + description: "Treat trailing commas as requests for vertical expansion.", + }), + ), + }, + { additionalProperties: false, title: "PythonFormatter" }, + ), + Type.Object( + { + kind: Type.Literal("javascript", { description: "Use Oxfmt's JavaScript parser." }), + ...javascriptFormatterProperties, + }, + { additionalProperties: false, title: "JavaScriptFormatter" }, + ), + Type.Object( + { + kind: Type.Literal("typescript", { description: "Use Oxfmt's TypeScript parser." }), + ...javascriptFormatterProperties, + }, + { additionalProperties: false, title: "TypeScriptFormatter" }, + ), + Type.Object( + { + kind: Type.Literal("bash", { + description: "Use shfmt's Bash formatter without minification.", + }), + command: formatterCommandSchema, + indentWidth: Type.Optional( + Type.Integer({ minimum: 1, default: 2, description: "Bash indentation width." }), + ), + caseIndent: Type.Optional( + Type.Boolean({ default: true, description: "Indent Bash case arms." }), + ), + spaceRedirects: Type.Optional( + Type.Boolean({ + default: true, + description: "Separate redirect operators from targets.", + }), + ), + binaryNextLine: Type.Optional( + Type.Boolean({ + default: false, + description: "Start continued Bash chains with their operator.", + }), + ), + }, + { additionalProperties: false, title: "BashFormatter" }, + ), +]); + +export type ScriptFormatterProfile = StaticDecode; + const scriptPreviewSchema = Type.Object( { headerLayout: Type.Union( @@ -321,8 +424,9 @@ const scriptPreviewSchema = Type.Object( shellOperatorPosition: Type.Union([Type.Literal("trailing"), Type.Literal("leading")], { description: "Place Bash chain operators before or after reflowed line breaks.", }), - formatters: Type.Record(Type.String(), formatterCommandSchema, { - description: "Commands that format script previews through stdin/stdout.", + formatters: Type.Record(Type.String(), scriptFormatterProfileSchema, { + description: + "Display-only profiles keyed by matching language id; expansion shows original source.", }), }, { @@ -360,6 +464,14 @@ export const extensionSettingsInput = { schemaId: "https://raw.githubusercontent.com/zigai/pi-tweaks/master/extensions/pi-glowup/config.schema.json", schema: settingsSchema, -} as const; + exampleSettings: { + scriptPreview: { + formatters: { + javascript: { kind: "javascript", command: ["oxfmt"], objectWrap: "collapse" }, + python: { kind: "python", command: ["tuff"], collections: "ruff-default" }, + }, + }, + }, +} as const satisfies ExtensionSettingsDefinitionInput; export default extensionSettingsInput; diff --git a/extensions/pi-glowup/src/settings.prevalidated.ts b/extensions/pi-glowup/src/settings.prevalidated.ts index e5373a0b..2492d47c 100644 --- a/extensions/pi-glowup/src/settings.prevalidated.ts +++ b/extensions/pi-glowup/src/settings.prevalidated.ts @@ -1,8 +1,8 @@ // Generated by pi-extension-settings. Do not edit. const artifact = { "formatVersion": 1, - "fingerprint": "json-sha256-v1:52e7b1e518e1a5ebcdfbdc0a73958f6cef68af7d419062760e6bea47ef8e6ebc", - "semanticFingerprint": "typebox-value-hash-v1:20ce17f2615529b2", + "fingerprint": "json-sha256-v1:94db173c94fbb3ebb2793b7a943d956acfe27f1f8b2047f28c12bd1b621e0748", + "semanticFingerprint": "typebox-value-hash-v1:cdc701c52f79f5fe", "defaultSettings": { "preserveTools": [], "mutations": { @@ -76,6 +76,26 @@ const artifact = { "shellOperatorPosition": "trailing", "formatters": {} } + }, + "exampleSettings": { + "scriptPreview": { + "formatters": { + "javascript": { + "kind": "javascript", + "command": [ + "oxfmt" + ], + "objectWrap": "collapse" + }, + "python": { + "kind": "python", + "command": [ + "tuff" + ], + "collections": "ruff-default" + } + } + } } } as const; diff --git a/extensions/pi-glowup/src/tools/built-in/bash/command.ts b/extensions/pi-glowup/src/tools/built-in/bash/command.ts index b326d147..bad5ee13 100644 --- a/extensions/pi-glowup/src/tools/built-in/bash/command.ts +++ b/extensions/pi-glowup/src/tools/built-in/bash/command.ts @@ -1,4 +1,11 @@ -import { parse, type CompoundList, type If, type Node, type ParsedScript } from "unbash"; +import { + parse, + type CompoundList, + type If, + type Node, + type ParsedScript, + type Redirect, +} from "unbash"; import { parseScriptInvocation, type ScriptInvocation } from "./invocation.ts"; @@ -27,9 +34,38 @@ function keywordIndex( start: number, end: number, keyword: string, + protectedRanges: readonly { readonly start: number; readonly end: number }[], ): number | undefined { - const match = new RegExp(`\\b${keyword}\\b`, "gu").exec(source.slice(start, end)); - return match === null ? undefined : start + match.index; + let index = start; + while (index < end) { + const protectedRange = protectedRanges.find( + (range) => index >= range.start && index < range.end, + ); + if (protectedRange !== undefined) { + index = protectedRange.end; + } else if (/[\s;&]/u.test(source[index] ?? "")) { + index += 1; + } else if (source[index] === "#") { + const newline = source.indexOf("\n", index); + if (newline < 0) return undefined; + + index = newline + 1; + } else if (source.startsWith("\\\n", index)) { + index += 2; + } else { + const keywordEnd = index + keyword.length; + if ( + keywordEnd <= end && + source.slice(index, keywordEnd) === keyword && + (keywordEnd === end || /[\s;&|()<>]/u.test(source[keywordEnd] ?? "")) + ) + return index; + + return undefined; + } + } + + return undefined; } function operatorIndex( @@ -44,6 +80,7 @@ function operatorIndex( class BashLayoutCollector { readonly #breakpoints = new Map(); + readonly #heredocs: Redirect[] = []; readonly #source: string; readonly #operatorPosition: ShellOperatorPosition; #structurallyComplex = false; @@ -96,6 +133,12 @@ class BashLayoutCollector { } } + keywordIndex(start: number, end: number, keyword: string): number | undefined { + const ranges = this.heredocRanges(); + if (ranges === undefined) return undefined; + return keywordIndex(this.#source, start, end, keyword, ranges); + } + walkIf(node: If, indent: number, depth: number): void { this.walkCompoundList(node.clause, indent + 1, depth + 1); this.add(node.then.pos, indent + 1); @@ -109,14 +152,14 @@ class BashLayoutCollector { let closingSearchStart = node.then.end; if (node.else !== undefined) { - const elseIndex = keywordIndex(this.#source, node.then.end, node.else.pos, "else"); + const elseIndex = this.keywordIndex(node.then.end, node.else.pos, "else"); if (elseIndex !== undefined) this.add(elseIndex, indent); this.add(node.else.pos, indent + 1); this.walkCompoundList(node.else, indent + 1, depth + 1); closingSearchStart = node.else.end; } - const closingIndex = keywordIndex(this.#source, closingSearchStart, node.end, "fi"); + const closingIndex = this.keywordIndex(closingSearchStart, node.end, "fi"); if (closingIndex !== undefined) this.add(closingIndex, indent); } @@ -128,14 +171,21 @@ class BashLayoutCollector { ): void { this.add(node.body.pos, indent + 1); this.walkCompoundList(node.body, indent + 1, depth + 1); - - const closingIndex = keywordIndex(this.#source, node.body.end, node.end, closingKeyword); + const closingIndex = this.keywordIndex(node.body.end, node.end, closingKeyword); if (closingIndex !== undefined) this.add(closingIndex, indent); } walkNode(node: Node, indent: number, depth: number): void { if (depth > MAX_LAYOUT_DEPTH) return; + if ("redirects" in node) { + this.#heredocs.push( + ...node.redirects.filter( + (redirect) => redirect.operator === "<<" || redirect.operator === "<<-", + ), + ); + } + switch (node.type) { case "Statement": this.walkNode(node.command, indent, depth + 1); @@ -207,7 +257,7 @@ class BashLayoutCollector { } const finalItemEnd = node.items.at(-1)?.end ?? node.word.end; - const closingIndex = keywordIndex(this.#source, finalItemEnd, node.end, "esac"); + const closingIndex = this.keywordIndex(finalItemEnd, node.end, "esac"); if (closingIndex !== undefined) this.add(closingIndex, indent); return; } @@ -244,26 +294,71 @@ class BashLayoutCollector { } } + heredocRanges(): Array<{ start: number; end: number }> | undefined { + const protectedRanges: Array<{ start: number; end: number }> = []; + const headerEnds = new Map(); + for (const heredoc of this.#heredocs.sort((left, right) => left.pos - right.pos)) { + const headerEnd = this.#source.indexOf("\n", heredoc.end); + if (headerEnd < 0 || heredoc.content === undefined) return undefined; + + const bodyStart = this.#source.indexOf( + heredoc.content, + headerEnds.get(headerEnd) ?? headerEnd + 1, + ); + if (bodyStart < 0) return undefined; + + const bodyEnd = bodyStart + heredoc.content.length; + const delimiterEnd = this.#source.indexOf("\n", bodyEnd); + let end = this.#source.length; + if (delimiterEnd >= 0) end = delimiterEnd + 1; + protectedRanges.push({ start: heredoc.pos, end }); + headerEnds.set(headerEnd, end); + } + + return protectedRanges; + } + render(): string | undefined { const breakpoints: DisplayBreakpoint[] = [...this.#breakpoints] .map(([index, indent]) => ({ index, indent })) .sort((left, right) => left.index - right.index); if (breakpoints.length === 0) return undefined; - const lines: string[] = []; + const protectedRanges = this.heredocRanges(); + if (protectedRanges === undefined) return undefined; + + let rendered = ""; let start = 0; - let indent = 0; + let changed = false; for (const breakpoint of breakpoints) { - const text = this.#source.slice(start, breakpoint.index).trim(); - if (text.length > 0) lines.push(`${INDENT.repeat(indent)}${text}`); + if ( + protectedRanges.some( + (range) => breakpoint.index > range.start && breakpoint.index < range.end, + ) + ) + continue; + + if (breakpoint.index <= start) { + rendered = rendered.replace(/[\t ]*$/u, INDENT.repeat(breakpoint.indent)); + continue; + } + + const segment = this.#source.slice(start, breakpoint.index); + start = breakpoint.index; - indent = breakpoint.indent; - } + if (/\n[\t ]*$/u.test(segment) || /^[\t ]*\r?\n/u.test(this.#source.slice(start))) { + rendered += segment; + } else { + rendered += `${segment.replace(/[\t ]+$/u, "")}\n${INDENT.repeat(breakpoint.indent)}`; - const tail = this.#source.slice(start).trim(); - if (tail.length > 0) lines.push(`${INDENT.repeat(indent)}${tail}`); + while (this.#source[start] === " " || this.#source[start] === "\t") start += 1; + changed = true; + } + } - return lines.length > 1 ? lines.join("\n") : undefined; + rendered += this.#source.slice(start); + if (changed) return rendered; + return undefined; } } @@ -277,11 +372,7 @@ function analyzeBashLayout( operatorPosition: ShellOperatorPosition, ): BashLayoutAnalysis { const unchanged = { reflowedCommand: undefined, structurallyComplex: false } as const; - if ( - command.includes("\n") || - command.trim().length === 0 || - command.length > MAX_LAYOUT_SOURCE_CHARACTERS - ) { + if (command.trim().length === 0 || command.length > MAX_LAYOUT_SOURCE_CHARACTERS) { return unchanged; } @@ -303,7 +394,7 @@ function analyzeBashLayout( }; } -/** Reflows a complete one-line Bash program while retaining its original non-whitespace text. */ +/** Reflows a complete Bash program while preserving quoted text and heredoc content. */ export function reflowBashCommand( command: string, operatorPosition: ShellOperatorPosition = "trailing", diff --git a/extensions/pi-glowup/src/tools/built-in/bash/display-profile.ts b/extensions/pi-glowup/src/tools/built-in/bash/display-profile.ts new file mode 100644 index 00000000..1110a242 --- /dev/null +++ b/extensions/pi-glowup/src/tools/built-in/bash/display-profile.ts @@ -0,0 +1,93 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join, resolve } from "node:path"; +import type { ScriptFormatterProfile } from "../../../settings-input.ts"; +import type { ScriptBlockFormatterOptions } from "./formatter.ts"; + +export type PreparedFormatterCommand = { + readonly executable: string; + readonly args: readonly string[]; + readonly cwd?: string; + readonly dispose?: () => Promise; +}; + +export async function prepareFormatterCommand( + profile: ScriptFormatterProfile, + options: ScriptBlockFormatterOptions, +): Promise { + const signal = options.signal; + const [command, ...args] = profile.command; + + if (command === undefined) throw new Error("Display formatter command is empty"); + if (profile.kind === "command") return { executable: command, args }; + + let executable = command; + if (!isAbsolute(command) && command.includes("/")) executable = resolve(command); + + const directory = await mkdtemp(join(tmpdir(), "pi-glowup-display-")); + const dispose = async () => rm(directory, { recursive: true, force: true }); + const width = String(options.targetWidth ?? 88); + + try { + signal?.throwIfAborted(); + + switch (profile.kind) { + case "python": { + const layout = profile.collections ?? "ruff-default"; + await writeFile( + join(directory, "tuff.toml"), + `[format.collections]\nlists = { layout = "${layout}" }\ndicts = { layout = "${layout}" }\ntuples = { layout = "${layout}" }\nsets = { layout = "${layout}" }\n`, + { mode: 0o600, signal }, + ); + args.push( + "format", + "--isolated", + "--line-length", + width, + "--config", + `indent-width = ${profile.indentWidth ?? 4}`, + "--config", + `format.skip-magic-trailing-comma = ${!(profile.magicTrailingComma ?? false)}`, + "--stdin-filename", + join(directory, "script.py"), + "-", + ); + + break; + } + case "javascript": + case "typescript": { + const config = join(directory, "oxfmt.json"); + + await writeFile( + config, + JSON.stringify({ + printWidth: Number(width), + tabWidth: profile.tabWidth ?? 2, + objectWrap: profile.objectWrap ?? "collapse", + }), + { mode: 0o600, signal }, + ); + + let filename = "script.js"; + if (profile.kind === "typescript") filename = "script.ts"; + args.push("--config", config, "--stdin-filepath", filename); + break; + } + case "bash": + args.push("-ln", "bash", "-i", String(profile.indentWidth ?? 2)); + + if (profile.caseIndent ?? true) args.push("-ci"); + if (profile.spaceRedirects ?? true) args.push("-sr"); + if (profile.binaryNextLine ?? false) args.push("-bn"); + args.push("-"); + break; + } + + signal?.throwIfAborted(); + return { executable, args, cwd: directory, dispose }; + } catch (cause: unknown) { + await dispose(); + throw cause; + } +} diff --git a/extensions/pi-glowup/src/tools/built-in/bash/format-preview.ts b/extensions/pi-glowup/src/tools/built-in/bash/format-preview.ts index f7d8e1a3..c9daebaf 100644 --- a/extensions/pi-glowup/src/tools/built-in/bash/format-preview.ts +++ b/extensions/pi-glowup/src/tools/built-in/bash/format-preview.ts @@ -7,6 +7,7 @@ import { boundedScriptPreview } from "./preview-store.ts"; import { parseScriptInvocation, type ScriptInvocation } from "./invocation.ts"; type ScriptPreviewSink = { + get(toolCallId: string): ScriptInvocation | undefined; set(toolCallId: string, preview: ScriptInvocation): void; }; @@ -21,13 +22,25 @@ export type FormatScriptPreviewOptions = { readonly invalidate?: () => void; }; +export function scriptForPreview( + command: string, + formatter: ScriptBlockFormatter | undefined, +): ScriptInvocation | undefined { + const script = parseScriptInvocation(command); + if (script !== undefined) return script; + if (formatter?.profileFor?.("bash") === undefined) return undefined; + + return { label: "Bash", language: "bash", code: command }; +} + /** Stores the cheap raw script preview for the command currently associated with a tool call. */ export function rememberRawScriptPreview( sink: ScriptPreviewSink, toolCallId: string, command: string, + formatter?: ScriptBlockFormatter, ): void { - const script = parseScriptInvocation(command); + const script = scriptForPreview(command, formatter); if (script === undefined) { return; } @@ -38,9 +51,9 @@ export function rememberRawScriptPreview( /** Formats and stores a script preview after the final bash command is known. */ export async function formatAndStoreScriptPreview( options: FormatScriptPreviewOptions, -): Promise { - const script = parseScriptInvocation(options.command); - if (options.formatter === undefined || script === undefined) return; +): Promise { + const script = scriptForPreview(options.command, options.formatter); + if (options.formatter === undefined || script === undefined) return false; const formatterOptions: ScriptBlockFormatterOptions = { ...(options.signal !== undefined && { signal: options.signal }), @@ -51,10 +64,13 @@ export async function formatAndStoreScriptPreview( options.formatter, formatterOptions, ); - if (formattedScript.code !== script.code && (options.isCurrent?.() ?? true)) { + const previousCode = options.sink.get(options.toolCallId)?.code ?? script.code; + if (formattedScript.code !== previousCode && (options.isCurrent?.() ?? true)) { options.sink.set(options.toolCallId, boundedScriptPreview(formattedScript)); options.invalidate?.(); } + + return formattedScript !== script; } /** Starts best-effort formatter work without blocking Pi tool preflight or result delivery. */ diff --git a/extensions/pi-glowup/src/tools/built-in/bash/formatter.ts b/extensions/pi-glowup/src/tools/built-in/bash/formatter.ts index 366233e5..674211a2 100644 --- a/extensions/pi-glowup/src/tools/built-in/bash/formatter.ts +++ b/extensions/pi-glowup/src/tools/built-in/bash/formatter.ts @@ -3,7 +3,13 @@ import { createHash } from "node:crypto"; import type { Readable, Writable } from "node:stream"; import type { ScriptInvocation } from "./invocation.ts"; import { jsonObjectParser, type JsonValue } from "../../../json-value.ts"; -import { jsonArrayParser, stringParser } from "../../../json-scalar.ts"; +import { stringParser } from "../../../json-scalar.ts"; +import { Value } from "typebox/value"; +import { + scriptFormatterProfileSchema, + type ScriptFormatterProfile, +} from "../../../settings-input.ts"; +import { prepareFormatterCommand, type PreparedFormatterCommand } from "./display-profile.ts"; export type ScriptBlockFormatterInput = { readonly label: string; @@ -17,12 +23,14 @@ export type ScriptBlockFormatterOptions = { readonly targetWidth?: number; }; -export type ScriptBlockFormatter = ( +export type ScriptBlockFormatter = (( input: ScriptBlockFormatterInput, options?: ScriptBlockFormatterOptions, -) => Promise; +) => Promise) & { + readonly profileFor?: (language: string) => ScriptFormatterProfile | undefined; +}; -export type ScriptFormatterCommands = ReadonlyMap; +export type ScriptFormatterCommands = ReadonlyMap; export type ScriptFormatterWarningReporter = (message: string) => void; export type ScriptFormatterParseOptions = { @@ -61,13 +69,14 @@ function isSafeFormatterCommandPart(value: JsonValue): value is string { return text !== undefined && text.trim().length > 0 && !text.includes("\0"); } -function formatterCommand(value: JsonValue): readonly string[] | undefined { - const parts = jsonArrayParser.parse(value); - if (parts === undefined || parts.length === 0 || !parts.every(isSafeFormatterCommandPart)) { +function formatterProfile(value: JsonValue): ScriptFormatterProfile | undefined { + try { + const profile = Value.Parse(scriptFormatterProfileSchema, value); + if (!profile.command.every(isSafeFormatterCommandPart)) return undefined; + return profile; + } catch { return undefined; } - - return parts; } function normalizeCode(code: string): string { @@ -100,7 +109,7 @@ export function parseScriptFormatterCommandsValue( return new Map(); } - const commands = new Map(); + const commands = new Map(); const invalidEntries: string[] = []; for (const [language, value] of Object.entries(record)) { if (language.trim().length === 0) { @@ -108,8 +117,8 @@ export function parseScriptFormatterCommandsValue( continue; } - const command = formatterCommand(value); - if (command === undefined) { + const command = formatterProfile(value); + if (command === undefined || (command.kind !== "command" && command.kind !== language)) { invalidEntries.push(language); continue; } @@ -158,6 +167,8 @@ async function runFormatterCommand( input: string, options: ScriptBlockFormatterOptions, release: () => void, + cwd: string | undefined, + reportFailure: (message: string) => void, ): Promise { if (options.signal?.aborted === true) { release(); @@ -187,12 +198,16 @@ async function runFormatterCommand( child = spawn(executable, [...finalArgs], { shell: false, signal: options.signal, + cwd, env, stdio: ["pipe", "pipe", "ignore"], }); } catch { release(); + + if (options.signal?.aborted !== true) reportFailure("could not start"); resolve(undefined); + return; } @@ -237,12 +252,26 @@ async function runFormatterCommand( }); child.stdin.on("error", fail); - child.on("error", fail); + child.on("error", (cause: unknown) => { + if (options.signal?.aborted !== true) { + let reason = "failed"; + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") { + reason = "executable not found"; + } + + reportFailure(reason); + } + + fail(); + }); + options.signal?.addEventListener("abort", fail, { once: true }); child.once("close", (code) => { options.signal?.removeEventListener("abort", fail); clearTimeout(escalation); + + if (code !== 0 && options.signal?.aborted !== true) reportFailure("failed"); finish(code === 0 ? stdout : undefined); release(); }); @@ -326,6 +355,7 @@ function startQueuedFormatters(): void { export function createCommandScriptFormatter( commands: ScriptFormatterCommands, + reportWarning: ScriptFormatterWarningReporter = console.warn, ): ScriptBlockFormatter | undefined { if (commands.size === 0) { return undefined; @@ -333,6 +363,13 @@ export function createCommandScriptFormatter( const cache = new Map(); let cacheBytes = 0; + const reportedFailures = new Set(); + const reportFailure = (language: string, reason: string): void => { + if (reportedFailures.has(language)) return; + + reportedFailures.add(language); + reportWarning(`[pi-glowup] ${language} display formatter ${reason}`); + }; const remember = (key: string, output: string): void => { const bytes = Buffer.byteLength(output, "utf8"); @@ -351,7 +388,7 @@ export function createCommandScriptFormatter( } }; - return async (input, options = {}) => { + const formatter: ScriptBlockFormatter = async (input, options = {}) => { if (options.signal?.aborted === true) return undefined; const command = commands.get(input.language); @@ -363,12 +400,6 @@ export function createCommandScriptFormatter( return undefined; } - const [executable, ...args] = command; - - if (executable === undefined) { - return undefined; - } - const targetWidth = input.targetWidth ?? options.targetWidth; const effectiveOptions: ScriptBlockFormatterOptions = targetWidth === undefined ? options : { ...options, targetWidth }; @@ -388,13 +419,44 @@ export function createCommandScriptFormatter( return undefined; } - const output = await runFormatterCommand( - executable, - args, - input.code, - effectiveOptions, - release, - ); + let prepared: PreparedFormatterCommand; + try { + prepared = await prepareFormatterCommand(command, effectiveOptions); + } catch { + release(); + + if (effectiveOptions.signal?.aborted !== true) + reportFailure(input.language, "profile could not be prepared"); + + return undefined; + } + + let resolveClosed = () => {}; + const closed = new Promise((resolve) => { + resolveClosed = resolve; + }); + const releaseClosed = (): void => { + release(); + resolveClosed(); + }; + let output: string | undefined; + try { + output = await runFormatterCommand( + prepared.executable, + prepared.args, + input.code, + effectiveOptions, + releaseClosed, + prepared.cwd, + (reason) => reportFailure(input.language, reason), + ); + } finally { + if (prepared.dispose !== undefined) { + await closed; + await prepared.dispose(); + } + } + if (output === undefined) { return undefined; } @@ -406,6 +468,8 @@ export function createCommandScriptFormatter( return formatted; }; + + return Object.assign(formatter, { profileFor: (language: string) => commands.get(language) }); } export async function formatScriptInvocation( diff --git a/extensions/pi-glowup/src/tools/built-in/bash/preview.ts b/extensions/pi-glowup/src/tools/built-in/bash/preview.ts index 49899a77..db2a5bd3 100644 --- a/extensions/pi-glowup/src/tools/built-in/bash/preview.ts +++ b/extensions/pi-glowup/src/tools/built-in/bash/preview.ts @@ -17,8 +17,7 @@ import { import { formatAndStoreScriptPreview, rememberRawScriptPreview, - scheduleFormattedScriptPreview, - type FormatScriptPreviewOptions, + scriptForPreview, } from "./format-preview.ts"; import type { ScriptBlockFormatter } from "./formatter.ts"; import { @@ -30,17 +29,30 @@ import { renderBashCommandCall, type BashCommandRenderOptions } from "./renderer import { StreamingScriptIdentityStore } from "./streaming-identity.ts"; import { renderExpandableSection } from "../../section-expansion.ts"; -function scriptPreviewTargetWidth(targetWidth?: number): number | undefined { - // SAFETY: process.stdout.columns is number | undefined at runtime in Node.js. - const stdoutColumns = process.stdout.columns as number | undefined; - return ( - targetWidth ?? (stdoutColumns !== undefined ? Math.max(20, stdoutColumns - 4) : undefined) - ); -} +type ScriptPreviewSchedule = { + readonly toolCallId: string; + readonly command: string; + readonly formatter: ScriptBlockFormatter | undefined; + readonly signal: AbortSignal | undefined; + readonly targetWidth?: number; + readonly completed?: boolean; + readonly isCurrent: () => boolean; + readonly invalidate: () => void; +}; + +type ScheduledPreview = { + readonly options: ScriptPreviewSchedule; + targetWidth: number | undefined; + controller: AbortController; + completion: Promise; + completed: boolean; + ready: boolean; +}; export function createNativeBashFeature() { const scriptPreviews = createScriptPreviewStore(); - const scheduledCommands = new Map(); + const scheduledCommands = new Map(); + const renderWidths = new Map(); const PARTIAL_BASH_COMMAND_PREVIEW_CHARS = 4_000; const streamingScriptIdentities = new StreamingScriptIdentityStore(); const bashRenderInvalidations = new Map void>(); @@ -102,10 +114,19 @@ export function createNativeBashFeature() { } const displayCommand = context.argsComplete ? command : partialBashCommandPreview(command); - const parsedScript = parseScriptInvocation(displayCommand); - const script = - scriptPreviews.get(context.toolCallId) ?? - (parsedScript === undefined ? undefined : boundedScriptPreview(parsedScript)); + const scheduled = scheduledCommands.get(context.toolCallId); + + if (context.result !== undefined && !context.isPartial) completePreview(context.toolCallId); + + const parsedScript = scriptForPreview(displayCommand, scheduled?.options.formatter); + const profile = scheduled?.options.formatter?.profileFor?.(parsedScript?.language ?? ""); + const profiled = profile !== undefined; + let script = scriptPreviews.get(context.toolCallId); + if (profiled && scheduled?.completed !== true) script = undefined; + + if (script === undefined && parsedScript !== undefined) + script = boundedScriptPreview(parsedScript); + const stableScript = context.argsComplete ? streamingScriptIdentities.finalize(context.toolCallId, script) : script; @@ -119,13 +140,30 @@ export function createNativeBashFeature() { shellOperatorPosition: shellOperatorPosition(), invalidate: context.invalidate, }; + if (stableScript !== undefined && context.argsComplete) { + bashOptions = { + ...bashOptions, + onContentWidth: (contentWidth, renderWidth) => + rememberRenderWidth(context.toolCallId, contentWidth, renderWidth), + }; + } + if (stableScript !== undefined) { bashOptions = { ...bashOptions, pureScriptOverride: stableScript }; } - return renderExpandableSection(context, "call", (expanded) => - renderBashCommandCall(theme, displayCommand, { ...bashOptions, expanded }), - ); + if (profile !== undefined && scheduled?.completed === true && scheduled.ready) { + bashOptions = { ...bashOptions, profile }; + } + + return renderExpandableSection(context, "call", (expanded) => { + let options = { ...bashOptions, expanded }; + if (expanded && profiled && parsedScript !== undefined) { + options = { ...options, pureScriptOverride: parsedScript, shellLayout: "preserve" }; + } + + return renderBashCommandCall(theme, displayCommand, options); + }); } function renderBashResult( @@ -135,6 +173,8 @@ export function createNativeBashFeature() { context: BuiltInRenderContext, headerLayout: () => ScriptPreviewHeaderLayout, ) { + if (!options.isPartial) completePreview(context.toolCallId); + const output = textOutput(result); const language = detectStructuredOutputLanguage(output); const script = parseScriptInvocation(commandField(context.args)); @@ -164,60 +204,143 @@ export function createNativeBashFeature() { function clear(): void { scriptPreviews.clear(); + + for (const scheduled of scheduledCommands.values()) scheduled.controller.abort(); scheduledCommands.clear(); + renderWidths.clear(); bashRenderInvalidations.clear(); streamingScriptIdentities.clear(); } - function remember(toolCallId: string, command: string): void { - rememberRawScriptPreview(scriptPreviews, toolCallId, command); + function remember(toolCallId: string, command: string, formatter?: ScriptBlockFormatter): void { + rememberRawScriptPreview(scriptPreviews, toolCallId, command, formatter); } - function schedule(options: { - readonly toolCallId: string; - readonly command: string; - readonly formatter: ScriptBlockFormatter | undefined; - readonly signal: AbortSignal | undefined; - readonly targetWidth?: number; - readonly isCurrent: () => boolean; - readonly invalidate: () => void; - }): boolean { - if ( - options.formatter !== undefined && - scheduledCommands.get(options.toolCallId) === options.command - ) { - return true; - } + function invalidatePreview(scheduled: ScheduledPreview): void { + bashRenderInvalidations.get(scheduled.options.toolCallId)?.(); + scheduled.options.invalidate(); + } - remember(options.toolCallId, options.command); - scheduledCommands.delete(options.toolCallId); + function completePreview(toolCallId: string): void { + const scheduled = scheduledCommands.get(toolCallId); + if (scheduled === undefined || scheduled.completed) return; - if (options.formatter !== undefined) { - scheduledCommands.set(options.toolCallId, options.command); - trimOldestMapEntries(scheduledCommands, MAX_BASH_RENDER_INVALIDATIONS); - } + scheduled.completed = true; + if (!scheduled.ready) return; + + queueMicrotask(() => { + if (scheduledCommands.get(toolCallId) === scheduled && scheduled.options.isCurrent()) { + invalidatePreview(scheduled); + } + }); + } - const columns = scriptPreviewTargetWidth(options.targetWidth); + async function formatPreview(scheduled: ScheduledPreview): Promise { + scheduled.controller.abort(); - const formatOptions: FormatScriptPreviewOptions = { + const controller = new AbortController(); + scheduled.controller = controller; + + const { options, targetWidth } = scheduled; + let signal = controller.signal; + if (options.signal !== undefined) signal = AbortSignal.any([signal, options.signal]); + + const formatted = await formatAndStoreScriptPreview({ sink: scriptPreviews, toolCallId: options.toolCallId, command: options.command, formatter: options.formatter, - ...(columns !== undefined && { targetWidth: columns }), - ...(options.signal !== undefined && { signal: options.signal }), + signal, + ...(targetWidth !== undefined && { targetWidth }), isCurrent: () => + !signal.aborted && options.isCurrent() && - scheduledCommands.get(options.toolCallId) === options.command, - invalidate: () => { - bashRenderInvalidations.get(options.toolCallId)?.(); - options.invalidate(); - }, + scheduledCommands.get(options.toolCallId) === scheduled && + scheduled.controller === controller, + }).catch(() => false); + + if ( + signal.aborted || + !options.isCurrent() || + scheduledCommands.get(options.toolCallId) !== scheduled || + scheduled.controller !== controller + ) + return; + + if (!formatted) return; + + scheduled.ready = true; + + const script = scriptPreviews.get(options.toolCallId); + const profile = options.formatter?.profileFor?.(script?.language ?? ""); + if (profile !== undefined && !scheduled.completed) return; + + invalidatePreview(scheduled); + } + + function rememberRenderWidth( + toolCallId: string, + contentWidth: number, + renderWidth: number, + ): void { + const previous = renderWidths.get(toolCallId); + if (previous?.renderWidth === renderWidth) { + contentWidth = Math.min(previous.contentWidth, contentWidth); + } + + renderWidths.delete(toolCallId); + renderWidths.set(toolCallId, { contentWidth, renderWidth }); + trimOldestMapEntries(renderWidths, MAX_BASH_RENDER_INVALIDATIONS); + + const scheduled = scheduledCommands.get(toolCallId); + if (scheduled === undefined || scheduled.targetWidth === contentWidth) return; + + scheduled.targetWidth = contentWidth; + queueMicrotask(() => { + if ( + scheduledCommands.get(toolCallId) !== scheduled || + scheduled.targetWidth !== contentWidth || + !scheduled.options.isCurrent() || + scheduled.options.signal?.aborted === true + ) + return; + + scheduled.completion = formatPreview(scheduled); + }); + } + + function schedule(options: ScriptPreviewSchedule): boolean { + const previous = scheduledCommands.get(options.toolCallId); + if (options.formatter !== undefined && previous?.options.command === options.command) + return true; + + previous?.controller.abort(); + remember(options.toolCallId, options.command, options.formatter); + scheduledCommands.delete(options.toolCallId); + + if (options.formatter === undefined) return false; + + const scheduled: ScheduledPreview = { + options, + targetWidth: renderWidths.get(options.toolCallId)?.contentWidth ?? options.targetWidth, + controller: new AbortController(), + completion: Promise.resolve(), + completed: options.completed ?? false, + ready: false, }; - scheduleFormattedScriptPreview(formatOptions); + scheduledCommands.set(options.toolCallId, scheduled); + + while (scheduledCommands.size > MAX_BASH_RENDER_INVALIDATIONS) { + const oldest = scheduledCommands.keys().next().value; + if (oldest === undefined) break; - return options.formatter !== undefined; + scheduledCommands.get(oldest)?.controller.abort(); + scheduledCommands.delete(oldest); + } + + scheduled.completion = formatPreview(scheduled); + return true; } async function restore(options: { @@ -229,27 +352,23 @@ export function createNativeBashFeature() { }): Promise { if (options.formatter === undefined) return; - const targetWidth = scriptPreviewTargetWidth(); let restored = 0; for (const call of options.calls) { if (options.signal.aborted || !options.isCurrent()) return; - if (parseScriptInvocation(call.command) === undefined) continue; + if (scriptForPreview(call.command, options.formatter) === undefined) continue; if (restored >= MAX_SCRIPT_PREVIEW_ENTRIES) return; restored += 1; - await formatAndStoreScriptPreview({ - sink: scriptPreviews, + schedule({ toolCallId: call.toolCallId, command: call.command, + completed: true, formatter: options.formatter, signal: options.signal, - ...(targetWidth !== undefined && { targetWidth }), isCurrent: options.isCurrent, - invalidate: () => { - bashRenderInvalidations.get(call.toolCallId)?.(); - options.invalidate(); - }, - }).catch(() => {}); + invalidate: options.invalidate, + }); + await scheduledCommands.get(call.toolCallId)?.completion; } } diff --git a/extensions/pi-glowup/src/tools/built-in/bash/renderer.ts b/extensions/pi-glowup/src/tools/built-in/bash/renderer.ts index 6c443004..25e8b73b 100644 --- a/extensions/pi-glowup/src/tools/built-in/bash/renderer.ts +++ b/extensions/pi-glowup/src/tools/built-in/bash/renderer.ts @@ -49,6 +49,13 @@ function scriptCallRenderOptions( renderOptions = { ...renderOptions, invalidate: options.invalidate }; } + if (options.onContentWidth !== undefined) { + renderOptions = { ...renderOptions, onContentWidth: options.onContentWidth }; + } + + if (options.profile !== undefined) + renderOptions = { ...renderOptions, profile: options.profile }; + return renderOptions; } @@ -119,6 +126,8 @@ export type BashCommandRenderOptions = { readonly shellOperatorPosition: ShellOperatorPosition; readonly pureScriptOverride?: ScriptInvocation; readonly invalidate?: () => void; + readonly onContentWidth?: (contentWidth: number, renderWidth: number) => void; + readonly profile?: ScriptCallRenderOptions["profile"]; }; /** Renders a clean standalone interpreter call or a losslessly reflowed Bash command. */ @@ -129,6 +138,15 @@ export function renderBashCommandCall( ): Component { const analysis = analyzeBashCommand(command, options.shellOperatorPosition); const pureScript = options.pureScriptOverride ?? analysis.pureScript; + if (pureScript?.language === "bash" && options.profile !== undefined) { + return renderFullShellCommand( + theme, + pureScript.code, + analyzeBashCommand(pureScript.code, options.shellOperatorPosition), + options, + ); + } + if (pureScript !== undefined) { return renderScriptCall( theme, diff --git a/extensions/pi-glowup/src/tools/built-in/bash/script-renderer.ts b/extensions/pi-glowup/src/tools/built-in/bash/script-renderer.ts index 5e475662..42575ab6 100644 --- a/extensions/pi-glowup/src/tools/built-in/bash/script-renderer.ts +++ b/extensions/pi-glowup/src/tools/built-in/bash/script-renderer.ts @@ -29,7 +29,10 @@ import { makeComponent, wrapPrefixedLine, } from "../../../rendering/component.ts"; -import { type Component } from "@earendil-works/pi-tui"; +import { type Component, visibleWidth } from "@earendil-works/pi-tui"; +import { parse, type WordPart } from "unbash"; +import { shellWords } from "./shell-words.ts"; +import type { ScriptFormatterProfile } from "../../../settings-input.ts"; export type ScriptPreviewHeaderLayout = "auto" | "inline" | "block"; @@ -71,6 +74,7 @@ type ShellCommandKind = "generic" | "interpreter" | "script" | "subcommands"; type ShellHighlightState = { readonly expectsCommand: boolean; + readonly expectsRedirectTarget: boolean; readonly expectingFlagValue: boolean; readonly commandKind: ShellCommandKind; readonly sawScriptOperand: boolean; @@ -175,6 +179,7 @@ const VALUE_SINGLE_DASH_LONG_FLAGS = new Set([ const initialShellHighlightState: ShellHighlightState = { expectsCommand: true, + expectsRedirectTarget: false, expectingFlagValue: false, commandKind: "generic", sawScriptOperand: false, @@ -231,6 +236,7 @@ type EmbeddedInlineHighlightRow = { function embeddedInlineHighlightRows( lines: ReadonlyArray, + invalidate: (() => void) | undefined, ): ReadonlyMap> { const source = lines.join("\n"); const scripts = embeddedInlineScripts(source); @@ -244,7 +250,7 @@ function embeddedInlineHighlightRows( const rows = new Map(); for (const script of scripts) { const code = source.slice(script.start, script.end); - const highlightedLines = highlightSyntaxCode(code, script.language); + const highlightedLines = highlightSyntaxCode(code, script.language, { invalidate }); let absoluteStart = script.start; for (const highlighted of highlightedLines) { let lineIndex = 0; @@ -297,9 +303,10 @@ function highlightShellLineWithEmbeddedCode( function highlightBashScriptPreviewLines( lines: ReadonlyArray, theme: GlowupRenderTheme, + invalidate: (() => void) | undefined, ): string[] { const highlighted: string[] = []; - const embeddedRows = embeddedInlineHighlightRows(lines); + const embeddedRows = embeddedInlineHighlightRows(lines, invalidate); let heredoc: BashHeredocHighlight | undefined; let heredocBody: string[] = []; @@ -308,7 +315,9 @@ function highlightBashScriptPreviewLines( return; } - highlighted.push(...highlightSyntaxCode(heredocBody.join("\n"), heredoc.language)); + highlighted.push( + ...highlightSyntaxCode(heredocBody.join("\n"), heredoc.language, { invalidate }), + ); heredocBody = []; } @@ -346,12 +355,17 @@ function highlightScriptPreviewLines( lines: ReadonlyArray, language: string, theme: GlowupRenderTheme, + invalidate: (() => void) | undefined, ): string[] { if (language === "bash") { - return highlightBashScriptPreviewLines(lines, theme); + return highlightBashScriptPreviewLines(lines, theme, invalidate); } - return [...highlightCodePreviewRuns(lines, (code) => highlightSyntaxCode(code, language))]; + return [ + ...highlightCodePreviewRuns(lines, (code) => + highlightSyntaxCode(code, language, { invalidate }), + ), + ]; } function scriptPreviewSyntaxLanguages(invocation: ScriptInvocation): readonly string[] { @@ -379,7 +393,11 @@ function scheduleScriptPreviewSyntaxLoads( invalidate: (() => void) | undefined, ): void { for (const language of scriptPreviewSyntaxLanguages(invocation)) { - scheduleCodeOutputSyntaxLoad({ language }, invalidate); + scheduleCodeOutputSyntaxLoad( + { language }, + invalidate, + language === invocation.language ? invocation.code : undefined, + ); } } @@ -387,7 +405,16 @@ function collapsedScriptPreview( invocation: ScriptInvocation, maxCodePreviewLines: number, showPrologueOmission: boolean, + profile?: ScriptFormatterProfile, ): ScriptPreview { + if (profile?.kind === "bash") return { code: invocation.code }; + + if (profile !== undefined && profile.kind !== "command") { + if (profile.imports === "show") return { code: invocation.code }; + + maxCodePreviewLines = 0; + } + if (trimEdgeBlankLines(invocation.code.split("\n")).length <= maxCodePreviewLines) { return { code: invocation.code }; } @@ -411,12 +438,13 @@ function scriptPreviewForRender( expanded: boolean, maxCodePreviewLines: number, showPrologueOmission: boolean, + profile?: ScriptFormatterProfile, ): ScriptPreview { if (expanded) { return { code: invocation.code }; } - return collapsedScriptPreview(invocation, maxCodePreviewLines, showPrologueOmission); + return collapsedScriptPreview(invocation, maxCodePreviewLines, showPrologueOmission, profile); } function retainedScriptInvocation( @@ -424,6 +452,7 @@ function retainedScriptInvocation( expanded: boolean, maxCodePreviewLines: number, showPrologueOmission: boolean, + profile?: ScriptFormatterProfile, ): ScriptInvocation { if (expanded) { return invocation; @@ -433,7 +462,8 @@ function retainedScriptInvocation( label: invocation.label, language: invocation.language, code: detachedScriptPreviewCode( - collapsedScriptPreview(invocation, maxCodePreviewLines, showPrologueOmission).code, + collapsedScriptPreview(invocation, maxCodePreviewLines, showPrologueOmission, profile) + .code, ), }; } @@ -502,6 +532,8 @@ export type ScriptCallRenderOptions = { readonly omittedHint?: string; readonly headerLayout?: ScriptPreviewHeaderLayout; readonly invalidate?: () => void; + readonly onContentWidth?: (contentWidth: number, renderWidth: number) => void; + readonly profile?: ScriptFormatterProfile; }; export function renderScriptCall( @@ -518,6 +550,7 @@ export function renderScriptCall( expanded, maxCodePreviewLines, showPrologueOmission, + options.profile, ); const omittedHint = options.omittedHint ?? toolExpandHint(); const headerLayoutOption = options.headerLayout ?? "auto"; @@ -531,14 +564,24 @@ export function renderScriptCall( expanded, maxCodePreviewLines, showPrologueOmission, + options.profile, ); if (preview.code.length === 0) { return wrapPrefixedLine("", width, header, " "); } - const fullLines = trimEdgeBlankLines(preview.code.split("\n")); - const fullHighlighted = highlightScriptPreviewLines(fullLines, retained.language, theme); + let fullLines = preview.code.split("\n"); + if (!expanded || options.profile === undefined) { + fullLines = trimEdgeBlankLines(fullLines); + } + + const fullHighlighted = highlightScriptPreviewLines( + fullLines, + retained.language, + theme, + options.invalidate, + ); const collapsedPreview = expanded ? undefined : collapsedPreviewLinesFromText( @@ -555,19 +598,35 @@ export function renderScriptCall( : collapsedPreview.lines; const rendered: string[] = []; const headerLayout = resolveScriptHeaderLayout(headerLayoutOption, retained, highlighted); + let prefixWidth = 4; + if (headerLayout === "inline") prefixWidth = visibleWidth(`${header} `); + options.onContentWidth?.(Math.max(1, width - Math.max(prefixWidth, 6)), width); + if (headerLayout === "block") { rendered.push(...wrapPrefixedLine("", width, header, " ")); } - let contentRows = 0; - let softWrapTruncated = false; - for (const [index, line] of highlighted.entries()) { + const wrappedLines = highlighted.map((line, index) => { const meta = isPreviewMetaLine(line); const styled = meta ? muted(theme, line) : line; const firstPrefix = headerLayout === "inline" && index === 0 ? `${header} ` : dim(theme, " │ "); - const wrapped = wrapScriptLine(theme, styled, width, firstPrefix); - if (expanded || meta) { + return { meta, wrapped: wrapScriptLine(theme, styled, width, firstPrefix) }; + }); + const hasTailOmission = highlighted.some( + (line) => isPreviewMetaLine(line) && !line.includes("import/setup lines omitted"), + ); + const totalContentRows = wrappedLines + .filter((line) => !line.meta) + .reduce((rows, line) => rows + line.wrapped.length, 0); + if (expanded || (!hasTailOmission && totalContentRows <= maxCodePreviewLines + 1)) { + return [...rendered, ...wrappedLines.flatMap((line) => line.wrapped)]; + } + + let contentRows = 0; + let softWrapTruncated = false; + for (const { meta, wrapped } of wrappedLines) { + if (meta) { rendered.push(...wrapped); continue; } @@ -604,11 +663,22 @@ function tokenizeShellLine(line: string): string[] { return []; } - return ( - line.match( - /"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|&&|\|\||2>>|2>|>>|\[\[|\]\]|[|;&<>{}!]|\s+|[^\s|;&<>{}!]+/g, - ) ?? [line] - ); + const gapTokens = (text: string): string[] => + text.match( + /(?:\d+)?(?:<<-|<<<|<<|>>|<>|<&|>&|>\||[<>])|&>>|&>|&&|\|\||\[\[|\]\]|[|;&{}!]|\s+|[^\s|;&<>{}!]+/gu, + ) ?? []; + + const tokens: string[] = []; + let cursor = 0; + for (const word of shellWords(line)) { + if (word.pos < cursor) continue; + tokens.push(...gapTokens(line.slice(cursor, word.pos)), word.text); + cursor = word.end; + } + + tokens.push(...gapTokens(line.slice(cursor))); + + return tokens; } function shellCommentStart(line: string): number | undefined { @@ -714,8 +784,52 @@ function styleShellFlagToken(theme: GlowupRenderTheme, token: string): string { return shellFlag(theme, token); } +function shellPart(theme: GlowupRenderTheme, part: WordPart, quoted: boolean): string { + switch (part.type) { + case "SingleQuoted": + case "AnsiCQuoted": + return shellString(theme, part.text); + case "DoubleQuoted": + case "LocaleString": { + let openingLength = 1; + if (part.type === "LocaleString") openingLength = 2; + + const childrenLength = part.parts.reduce( + (length, child) => length + child.text.length, + 0, + ); + + return `${shellString(theme, part.text.slice(0, openingLength))}${part.parts.map((child) => shellPart(theme, child, true)).join("")}${shellString(theme, part.text.slice(openingLength + childrenLength))}`; + } + case "SimpleExpansion": + case "ParameterExpansion": + case "CommandExpansion": + case "ArithmeticExpansion": + return theme.fg("syntaxVariable", part.text); + case "BraceExpansion": + case "ExtendedGlob": + case "Literal": + case "ProcessSubstitution": + break; + } + + if (quoted) return shellString(theme, part.text); + return shellText(theme, part.text); +} + function shellValue(theme: GlowupRenderTheme, token: string): string { - return isQuotedShellString(token) ? shellString(theme, token) : shellText(theme, token); + try { + const command = parse(`: ${token}`).commands[0]?.command; + if (command?.type === "Command") { + const parts = command.suffix[0]?.parts; + if (parts !== undefined) + return parts.map((part) => shellPart(theme, part, false)).join(""); + } + } catch { + return shellText(theme, token); + } + + return shellText(theme, token); } function shellFlagConsumesValue(token: string): boolean { @@ -767,25 +881,21 @@ function styleShellToken( return { styled: token, state }; } - if ( - [ - "|", - "||", - "&&", - "&", - ";", - ">", - ">>", - "<", - "2>", - "2>>", - "{", - "}", - "!", - "[[", - "]]", - ].includes(token) - ) { + if (/^(?:\d+)?(?:<<-|<<<|<<|>>|<>|<&|>&|>\||[<>])$|^&>>?$/u.test(token)) { + return { + styled: shellOperator(theme, token), + state: { ...state, expectsRedirectTarget: true }, + }; + } + + if (state.expectsRedirectTarget) { + return { + styled: shellValue(theme, token), + state: { ...state, expectsRedirectTarget: false }, + }; + } + + if (["|", "||", "&&", "&", ";", "{", "}", "!", "[[", "]]"].includes(token)) { return { styled: shellOperator(theme, token), state: initialShellHighlightState }; } @@ -815,12 +925,16 @@ function styleShellToken( }; } - if (/^[A-Za-z_][A-Za-z0-9_]*=.*/.test(token)) { - return { styled: shellString(theme, token), state }; + const assignment = /^(?[A-Za-z_][A-Za-z0-9_]*)(?\+?=)/u.exec(token); + if (assignment?.groups !== undefined) { + return { + styled: `${theme.fg("syntaxVariable", assignment.groups.name ?? "")}${shellOperator(theme, assignment.groups.operator ?? "")}${shellValue(theme, token.slice(assignment[0].length))}`, + state, + }; } - if (isQuotedShellString(token)) { - return { styled: shellString(theme, token), state: shellStateAfterOperand(state) }; + if (isQuotedShellString(token) || token.includes("$")) { + return { styled: shellValue(theme, token), state: shellStateAfterOperand(state) }; } if (state.expectsCommand) { @@ -832,6 +946,7 @@ function styleShellToken( styled: shellCommand(theme, token), state: { expectsCommand: false, + expectsRedirectTarget: false, expectingFlagValue: false, commandKind: shellCommandKind(token), sawScriptOperand: false, diff --git a/extensions/pi-glowup/src/tools/built-in/bash/shell-words.ts b/extensions/pi-glowup/src/tools/built-in/bash/shell-words.ts new file mode 100644 index 00000000..720ac5c2 --- /dev/null +++ b/extensions/pi-glowup/src/tools/built-in/bash/shell-words.ts @@ -0,0 +1,85 @@ +import { parse, type Node, type Redirect, type Word } from "unbash"; + +type ShellWord = Pick; + +export function shellWords(source: string): readonly ShellWord[] { + const words: ShellWord[] = []; + const redirects = (items: readonly Redirect[]): void => { + for (const redirect of items) { + if (redirect.target !== undefined) words.push(redirect.target); + } + }; + + const walk = (node: Node): void => { + switch (node.type) { + case "Statement": + redirects(node.redirects); + walk(node.command); + return; + case "Command": + words.push(...node.prefix, ...node.suffix); + + if (node.name !== undefined) words.push(node.name); + redirects(node.redirects); + + return; + case "Pipeline": + case "AndOr": + case "CompoundList": + for (const child of node.commands) walk(child); + + return; + case "If": + walk(node.clause); + walk(node.then); + + if (node.else !== undefined) walk(node.else); + return; + case "While": + walk(node.clause); + walk(node.body); + return; + case "For": + case "Select": + words.push(node.name, ...node.wordlist); + walk(node.body); + return; + case "Function": + words.push(node.name); + redirects(node.redirects); + walk(node.body); + return; + case "Coproc": + if (node.name !== undefined) words.push(node.name); + redirects(node.redirects); + walk(node.body); + + return; + case "ArithmeticFor": + case "BraceGroup": + case "Subshell": + walk(node.body); + return; + case "Case": + words.push(node.word); + + for (const item of node.items) { + words.push(...item.pattern); + walk(item.body); + } + + return; + case "ArithmeticCommand": + case "TestCommand": + return; + } + }; + + try { + for (const statement of parse(source).commands) walk(statement); + } catch { + return []; + } + + return words.sort((left, right) => left.pos - right.pos); +} diff --git a/extensions/pi-glowup/src/tools/external/codemode.ts b/extensions/pi-glowup/src/tools/external/codemode.ts index 4a7fc2c9..fc50dcb0 100644 --- a/extensions/pi-glowup/src/tools/external/codemode.ts +++ b/extensions/pi-glowup/src/tools/external/codemode.ts @@ -1,5 +1,6 @@ import { makeComponent, wrapPrefixedLine, toolExpandHint } from "../../rendering/component.ts"; -import { formatScriptInvocation, type ScriptBlockFormatter } from "../built-in/bash/formatter.ts"; +import type { ScriptBlockFormatter } from "../built-in/bash/formatter.ts"; +import { omitLeadingImportPrologue } from "../built-in/bash/prologue.ts"; import { PreviewStore } from "../built-in/file-previews.ts"; import { renderGlowupOutput } from "../../rendering/output.ts"; import { @@ -27,104 +28,233 @@ const SCRIPT_HEADER = /^Script (?:completed|failed)\nWall time [\d.]+ seconds\nO const VISIBLE_CALLS = 8; const MAX_CALLS = 256; -type FormattedCode = { readonly source: string; readonly code: string }; - -export function createCodemodePreview() { - const previews = new PreviewStore({ +type CodemodeSchedule = { + readonly toolCallId: string; + readonly code: string; + readonly formatter: ScriptBlockFormatter | undefined; + readonly isCurrent: () => boolean; + readonly invalidate: () => void; + readonly completed?: boolean; + readonly signal?: AbortSignal; +}; + +type CodemodeInvalidation = { callback: () => void; readonly invalidate: () => void }; + +type CodemodeWork = { + options: CodemodeSchedule; + width: number | undefined; + controller: AbortController; + completed: boolean; + ready: boolean; +}; + +export type CodemodePreviewReadOptions = { + readonly contentWidth: number; + readonly completed: boolean; + readonly expanded: boolean; +}; + +export function createCodemodePreview(formatterAccessor?: () => ScriptBlockFormatter | undefined) { + const previews = new PreviewStore<{ readonly work: CodemodeWork; readonly code: string }>({ maxEntries: 300, maxBytes: 4 * 1024 * 1024, - measureBytes: ({ source, code }) => - Buffer.byteLength(source, "utf8") + Buffer.byteLength(code, "utf8"), + measureBytes: ({ code }) => Buffer.byteLength(code, "utf8"), }); - const invalidations = new Map void>(); - const scheduled = new Map< - string, - { - code: string; - isCurrent: () => boolean; - invalidate: () => void; - } - >(); + const invalidations = new Map(); + const scheduled = new Map(); let generation = 0; - let controller = new AbortController(); - return { - observe(toolCallId: string, invalidate: (() => void) | undefined): void { - if (invalidate === undefined) return; + function invalidate(work: CodemodeWork): void { + invalidations.get(work.options.toolCallId)?.invalidate(); + work.options.invalidate(); + } + + function complete(work: CodemodeWork): void { + if (work.completed) return; + + work.completed = true; + if (work.ready) + queueMicrotask(() => { + if (scheduled.get(work.options.toolCallId) === work && work.options.isCurrent()) + invalidate(work); + }); + } + + async function format(work: CodemodeWork): Promise { + if (work.width === undefined) return; + + work.controller.abort(); + + const controller = new AbortController(); + work.controller = controller; + + const { options, width } = work; + let signal = controller.signal; + if (options.signal !== undefined) signal = AbortSignal.any([signal, options.signal]); + + const code = await options + .formatter?.( + { label: "Codemode", language: "javascript", code: options.code }, + { signal, targetWidth: width }, + ) + .catch(() => undefined); + if ( + code === undefined || + signal.aborted || + scheduled.get(options.toolCallId) !== work || + work.controller !== controller || + !work.options.isCurrent() + ) + return; + + previews.set(options.toolCallId, { work, code }); + work.ready = true; + if (work.completed) invalidate(work); + } + + const feature = { + observe( + toolCallId: string, + callback: (() => void) | undefined, + completed = false, + ): (() => void) | undefined { + const work = scheduled.get(toolCallId); + if (completed && work !== undefined) complete(work); + if (callback === undefined) return invalidations.get(toolCallId)?.invalidate; + + let observation = invalidations.get(toolCallId); + if (observation === undefined) { + const owner: CodemodeInvalidation = { + callback, + invalidate: () => owner.callback(), + }; + + observation = owner; + } else observation.callback = callback; invalidations.delete(toolCallId); - invalidations.set(toolCallId, invalidate); + invalidations.set(toolCallId, observation); if (invalidations.size > 300) { const oldest = invalidations.keys().next().value; if (oldest !== undefined) invalidations.delete(oldest); } + + return observation.invalidate; }, - get(toolCallId: string, source: string): string | undefined { + get( + toolCallId: string, + source: string, + read?: CodemodePreviewReadOptions, + ): string | undefined { + let work = scheduled.get(toolCallId); + if ((work === undefined || work.options.code !== source) && read?.completed === true) { + const formatter = formatterAccessor?.(); + const callback = invalidations.get(toolCallId)?.invalidate; + if (formatter !== undefined && callback !== undefined) { + const currentGeneration = generation; + + feature.schedule({ + toolCallId, + code: source, + formatter, + completed: true, + isCurrent: () => generation === currentGeneration, + invalidate: callback, + }); + work = scheduled.get(toolCallId); + } + } + + if (work === undefined || work.options.code !== source || !work.options.isCurrent()) + return undefined; + + if (read !== undefined) { + if (read.completed) complete(work); + + if (work.width !== read.contentWidth) { + work.controller.abort(); + work.ready = false; + work.width = read.contentWidth; + queueMicrotask(() => { + if ( + scheduled.get(toolCallId) === work && + work.options.isCurrent() && + work.width === read.contentWidth + ) + void format(work); + }); + } + + if (read.expanded) return undefined; + } + + if (!work.completed || !work.ready) return undefined; + const preview = previews.get(toolCallId); - return preview?.source === source ? preview.code : undefined; - }, - schedule(options: { - readonly toolCallId: string; - readonly code: string; - readonly formatter: ScriptBlockFormatter | undefined; - readonly isCurrent: () => boolean; - readonly invalidate: () => void; - }): void { - if (options.formatter === undefined) return; + if (preview?.work !== work) return undefined; + + const profile = work.options.formatter?.profileFor?.("javascript"); + if ( + profile === undefined || + profile.kind === "command" || + profile.kind === "bash" || + profile.imports === "show" + ) + return preview.code; + const omission = omitLeadingImportPrologue(preview.code, "javascript", 0); + return omission !== undefined + ? `… ${omission.omittedLines} import/setup lines omitted\n${omission.code}` + : preview.code; + }, + schedule(options: CodemodeSchedule): void { const previous = scheduled.get(options.toolCallId); - scheduled.set(options.toolCallId, { - code: options.code, - isCurrent: options.isCurrent, - invalidate: options.invalidate, - }); + if ( + previous?.options.code === options.code && + previous.options.formatter === options.formatter + ) { + previous.options = options; + + if (options.completed === true) complete(previous); + return; + } + + previous?.controller.abort(); + scheduled.delete(options.toolCallId); + + if (options.formatter === undefined) return; - if (previous?.code === options.code) return; + const work: CodemodeWork = { + options, + width: previous?.width, + controller: new AbortController(), + completed: options.completed ?? false, + ready: false, + }; + scheduled.set(options.toolCallId, work); if (scheduled.size > 300) { const oldest = scheduled.keys().next().value; - if (oldest !== undefined) scheduled.delete(oldest); + if (oldest !== undefined) { + scheduled.get(oldest)?.controller.abort(); + scheduled.delete(oldest); + } } - const currentGeneration = generation; - const columns = process.stdout.columns; - const targetWidth = Number.isFinite(columns) ? Math.max(20, columns - 4) : undefined; - - void formatScriptInvocation( - { label: "Codemode", language: "javascript", code: options.code }, - options.formatter, - { signal: controller.signal, ...(targetWidth !== undefined && { targetWidth }) }, - ) - .then((formatted) => { - const current = scheduled.get(options.toolCallId); - if ( - formatted.code === options.code || - generation !== currentGeneration || - current?.code !== options.code || - !current.isCurrent() - ) - return; - - previews.set(options.toolCallId, { - source: options.code, - code: formatted.code, - }); - - invalidations.get(options.toolCallId)?.(); - current.invalidate(); - }) - .catch(() => {}); + void format(work); }, clear(): void { generation += 1; - controller.abort(); - controller = new AbortController(); + + for (const work of scheduled.values()) work.controller.abort(); previews.clear(); invalidations.clear(); scheduled.clear(); }, }; + + return feature; } function formatCost(cost: number): string { @@ -280,22 +410,20 @@ function callRows( export function createCodemodeRenderer( labelMode: ToolLabelMode = "static", - preview?: Pick, "get" | "observe">, + preview: Pick< + ReturnType, + "get" | "observe" + > = createCodemodePreview(), ): ThirdPartyToolRenderer { return { renderCall(args, theme, context) { const record = jsonObjectParser.parse(args); const sourceCode = record === undefined ? undefined : getString(record, "code"); - preview?.observe(context.toolCallId, context.invalidate); - - const code = - sourceCode === undefined - ? undefined - : (preview?.get(context.toolCallId, sourceCode) ?? sourceCode); - if (code !== undefined) { - scheduleCodeOutputSyntaxLoad({ language: "javascript" }, context.invalidate, code); - } + const completed = + context.phase === "complete" || + (context.result !== undefined && !context.isPartial); + const invalidate = preview.observe(context.toolCallId, context.invalidate, completed); return renderExpandableSection(context, "call", (expanded) => { const header = renderGlowupCall(theme, { @@ -306,21 +434,37 @@ export function createCodemodeRenderer( completed: "Ran code", }), }); - const source = renderGlowupOutput(theme, code, { - expanded, - mode: "head", - maxPreviewLines: 8, - prefixFirst: dim(theme, " │ "), - prefixRest: dim(theme, " │ "), - dimContent: false, - noOutputLabel: null, - syntax: { language: "javascript" }, - }); - return makeComponent((width) => [...header.render(width), ...source.render(width)]); + return makeComponent((width) => { + const code = + sourceCode === undefined + ? undefined + : (preview.get(context.toolCallId, sourceCode, { + contentWidth: Math.max(1, width - 4), + completed, + expanded, + }) ?? sourceCode); + if (code !== undefined) + scheduleCodeOutputSyntaxLoad({ language: "javascript" }, invalidate, code); + + const source = renderGlowupOutput(theme, code, { + expanded, + mode: "head", + maxPreviewLines: 8, + prefixFirst: dim(theme, " │ "), + prefixRest: dim(theme, " │ "), + dimContent: false, + noOutputLabel: null, + syntax: { language: "javascript", invalidate }, + }); + + return [...header.render(width), ...source.render(width)]; + }); }); }, renderResult(result, options, theme, context) { + preview.observe(context.toolCallId, context.invalidate, !options.isPartial); + const text = options.isPartial ? undefined : scriptOutput(result); return renderExpandableSection( diff --git a/extensions/pi-glowup/src/tools/types.ts b/extensions/pi-glowup/src/tools/types.ts index f034f17f..5a9cddd1 100644 --- a/extensions/pi-glowup/src/tools/types.ts +++ b/extensions/pi-glowup/src/tools/types.ts @@ -71,8 +71,21 @@ export type ThirdPartyToolRenderingOptions = { readonly mutationSettings?: MutationSettings; readonly codemodePreview?: { - get(toolCallId: string, source: string): string | undefined; - observe(toolCallId: string, invalidate: (() => void) | undefined): void; + get( + toolCallId: string, + source: string, + options?: { + readonly contentWidth: number; + readonly completed: boolean; + readonly expanded: boolean; + }, + ): string | undefined; + + observe( + toolCallId: string, + invalidate: (() => void) | undefined, + completed?: boolean, + ): (() => void) | undefined; }; }; diff --git a/extensions/pi-glowup/test/config/load.test.ts b/extensions/pi-glowup/test/config/load.test.ts index 1db95a36..7e3d182a 100644 --- a/extensions/pi-glowup/test/config/load.test.ts +++ b/extensions/pi-glowup/test/config/load.test.ts @@ -474,7 +474,10 @@ describe("glowup config", () => { projectLanguageDetection: { enabled: true }, }, patches: { workingWidgetSpacing: false, markdownSyntax: false }, - scriptPreview: { maxCodePreviewLines: 4, formatters: { python: ["black", "-"] } }, + scriptPreview: { + maxCodePreviewLines: 4, + formatters: { python: { kind: "command", command: ["black", "-"] } }, + }, }), ); @@ -553,7 +556,10 @@ describe("glowup config", () => { projectLanguageDetection: { enabled: true }, }, patches: { workingWidgetSpacing: false, markdownSyntax: false }, - scriptPreview: { maxCodePreviewLines: 4, formatters: { python: ["black", "-"] } }, + scriptPreview: { + maxCodePreviewLines: 4, + formatters: { python: { kind: "command", command: ["black", "-"] } }, + }, }), ); @@ -574,7 +580,11 @@ describe("glowup config", () => { expect(config.scriptHeaderLayout).toBe("block"); expect(config.scriptMaxCodePreviewLines).toBe(4); - expect(config.scriptFormatters.get("python")).toEqual(["black", "-"]); + expect(config.scriptFormatters.get("python")).toEqual({ + kind: "command", + command: ["black", "-"], + }); + expect(config.toolCallIndicator).toEqual({ symbol: "·", bold: true }); expect(config.toolLabels.mode).toBe("static"); expect(config.writePreview.movingViewport).toBe(true); diff --git a/extensions/pi-glowup/test/package.test.ts b/extensions/pi-glowup/test/package.test.ts index 7255f392..4cc1d2d0 100644 --- a/extensions/pi-glowup/test/package.test.ts +++ b/extensions/pi-glowup/test/package.test.ts @@ -1,9 +1,10 @@ import { execFileSync, spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { copyFile, cp, mkdir, mkdtemp, readdir, rm, symlink, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { homedir, tmpdir } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { stripVTControlCharacters } from "node:util"; import { readModuleGraph } from "./module-graph.ts"; import { describe, expect, it, onTestFinished } from "vitest"; import Type, { type Static } from "typebox"; @@ -133,6 +134,89 @@ describe("package manifest", () => { expect(manifest.devDependencies?.typebox).toBeDefined(); }); + it("tokenizes long lines from the packed worker with extension-local host peers absent", async () => { + const packageRoot = resolve(import.meta.dirname, ".."); + const root = await mkdtemp(join(homedir(), ".cache", "glowup-worker-")); + onTestFinished(async () => { + await rm(root, { recursive: true, force: true }); + }); + const buildRoot = join(root, "build"); + await mkdir(buildRoot); + + for (const file of ["package.json", "config.schema.json", "LICENSE", "README.md"]) + await copyFile(join(packageRoot, file), join(buildRoot, file)); + + for (const folder of ["src", "scripts", "themes"]) + await cp(join(packageRoot, folder), join(buildRoot, folder), { recursive: true }); + + for (const dependency of [ + "esbuild", + "typescript", + "@pierre/diffs", + "ansi-styles", + "unbash", + "shiki", + ]) { + const destination = join(buildRoot, "node_modules", dependency); + await mkdir(dirname(destination), { recursive: true }); + await symlink(resolvedPackageRoot(dependency), destination, "dir"); + } + + execFileSync(process.execPath, [join(buildRoot, "scripts", "build.mjs")], { + cwd: buildRoot, + stdio: "pipe", + }); + + execFileSync("npm", ["pack", "--ignore-scripts", "--pack-destination", root], { + cwd: buildRoot, + stdio: "pipe", + }); + const archive = (await readdir(root)).find((name) => name.endsWith(".tgz")); + if (archive === undefined) throw new Error("Missing tarball"); + + const consumerRoot = join(root, "consumer"); + const installedRoot = join(consumerRoot, "node_modules", "@zigai", "pi-glowup"); + await mkdir(installedRoot, { recursive: true }); + execFileSync("tar", [ + "-xzf", + join(root, archive), + "-C", + installedRoot, + "--strip-components=1", + ]); + await symlink( + resolvedPackageRoot("shiki"), + join(consumerRoot, "node_modules", "shiki"), + "dir", + ); + const source = `const before = 1;\n/* ${"note ".repeat(1_800)}\nconst hidden = 2;\n*/ const after = 3;`; + const consumer = join(consumerRoot, "consumer.mjs"); + await writeFile( + consumer, + `import { Worker } from "node:worker_threads"; +import fs from "node:fs"; +const theme = JSON.parse(fs.readFileSync(new URL("./node_modules/@zigai/pi-glowup/themes/darker-modern-theme.json", import.meta.url), "utf8")); +const worker = new Worker(new URL("./node_modules/@zigai/pi-glowup/dist/src/tokenize-worker.mjs", import.meta.url), { execArgv: [] }); +try { + const response = new Promise((resolve, reject) => { worker.once("message", resolve); worker.once("error", reject); }); + worker.postMessage({ id: 1, code: ${JSON.stringify(source)}, language: "javascript", brackets: true, theme }); + console.log(JSON.stringify(await response)); +} finally { await worker.terminate(); } +`, + ); + const raw: unknown = JSON.parse( + execFileSync(process.execPath, [consumer], { cwd: consumerRoot, encoding: "utf8" }), + ); + const result = Value.Parse( + Type.Object({ id: Type.Number(), lines: Type.Array(Type.String()) }), + raw, + ); + expect(result.lines[1]).toContain("\u001b["); + expect(result.lines[2]).toContain("\u001b[38;2;106;153;85mconst hidden"); + expect(result.lines[3]).toContain("\u001b[38;2;86;156;214mconst"); + expect(result.lines.map((line) => stripVTControlCharacters(line)).join("\n")).toBe(source); + }, 30_000); + it("publishes a callable root factory with checked consumer types", async () => { const packageRoot = resolve(import.meta.dirname, ".."); const root = await mkdtemp(join(tmpdir(), "glowup-root-types-")); diff --git a/extensions/pi-glowup/test/pi/index-lifecycle.test.ts b/extensions/pi-glowup/test/pi/index-lifecycle.test.ts index 7dd592d0..2bde30e8 100644 --- a/extensions/pi-glowup/test/pi/index-lifecycle.test.ts +++ b/extensions/pi-glowup/test/pi/index-lifecycle.test.ts @@ -2,7 +2,8 @@ import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; +import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; import { createEditTool, @@ -558,7 +559,10 @@ describe("extension lifecycle", () => { const root = mkdtempSync(join(tmpdir(), "pi-glowup-lifecycle-")); process.env[AGENT_DIR_ENV] = join(root, "agent"); process.env[SCRIPT_FORMATTERS_ENV] = JSON.stringify({ - python: [process.execPath, "-e", "setTimeout(() => {}, 1000)"], + python: { + kind: "command", + command: [process.execPath, "-e", "setTimeout(() => {}, 1000)"], + }, }); const pi = new FakeExtensionApi(); @@ -575,11 +579,16 @@ describe("extension lifecycle", () => { const root = mkdtempSync(join(tmpdir(), "pi-glowup-restored-node-")); process.env[AGENT_DIR_ENV] = join(root, "agent"); process.env[SCRIPT_FORMATTERS_ENV] = JSON.stringify({ - javascript: [ - process.execPath, - "-e", - "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>process.stdout.write(s.replace('; ', ';\\n')))", - ], + javascript: { + kind: "javascript", + command: [ + join( + dirname(createRequire(import.meta.url).resolve("oxfmt/package.json")), + "bin", + "oxfmt", + ), + ], + }, }); const command = `node -e 'const item = "RESTORED_NODE"; console.log(item)'`; const call = { diff --git a/extensions/pi-glowup/test/pi/markdown-patch.test.ts b/extensions/pi-glowup/test/pi/markdown-patch.test.ts index e04a7d84..d6989e79 100644 --- a/extensions/pi-glowup/test/pi/markdown-patch.test.ts +++ b/extensions/pi-glowup/test/pi/markdown-patch.test.ts @@ -1,5 +1,11 @@ -import { describe, expect, it } from "vitest"; -import type { MarkdownTheme } from "@earendil-works/pi-tui"; +import { describe, expect, it, onTestFinished, vi } from "vitest"; +import { Markdown, type MarkdownTheme } from "@earendil-works/pi-tui"; +import { resolve } from "node:path"; +import { + initializeSyntaxHighlighting, + disposeSyntaxHighlighting, + settleSyntaxHighlights, +} from "../../src/rendering/syntax/highlighter.ts"; import { configureMarkdownSyntaxPatch } from "../../src/pi/patches/markdown-syntax.ts"; function makeMarkdownTheme(): MarkdownTheme { @@ -26,6 +32,78 @@ type FakeMarkdownInstance = { }; describe("markdown syntax patch", () => { + it("loads and repaints a language not preloaded for a real Markdown fence", async () => { + onTestFinished(async () => { + configureMarkdownSyntaxPatch(false); + await disposeSyntaxHighlighting(); + }); + await initializeSyntaxHighlighting( + { PI_GLOWUP_SYNTAX_THEME: resolve("themes/darker-modern-theme.json") }, + { preloadLanguages: ["javascript"] }, + ); + configureMarkdownSyntaxPatch(true); + const markdown = new Markdown( + "```sql\nSELECT item FROM entries WHERE item = 1;\n```", + 0, + 0, + makeMarkdownTheme(), + ); + expect(markdown.render(80).join("\n")).not.toContain("38;2"); + await vi.waitFor(() => expect(markdown.render(80).join("\n")).toContain("38;2")); + }); + + it("repaints a cached real Markdown block after full long-line colors complete", async () => { + onTestFinished(async () => { + configureMarkdownSyntaxPatch(false); + await disposeSyntaxHighlighting(); + }); + await initializeSyntaxHighlighting( + { PI_GLOWUP_SYNTAX_THEME: resolve("themes/darker-modern-theme.json") }, + { preloadLanguages: ["javascript"] }, + ); + configureMarkdownSyntaxPatch(true); + const code = `const before = 1;\n/* ${"note ".repeat(1_800)}\nconst hidden = 2;\n*/ const after = 3;`; + const markdown = new Markdown( + `\`\`\`javascript\n${code}\n\`\`\``, + 0, + 0, + makeMarkdownTheme(), + ); + const first = markdown.render(120).join("\n"); + expect(first).not.toContain("\u001b[38;2;86;156;214mconst"); + await settleSyntaxHighlights(); + const colored = markdown.render(120).join("\n"); + expect(colored).toContain("\u001b[38;2;106;153;85mconst hidden"); + expect(colored).toContain("\u001b[38;2;86;156;214mconst"); + expect(colored).toContain("after"); + }); + + it("does not repaint replaced real Markdown source with an obsolete worker result", async () => { + onTestFinished(async () => { + configureMarkdownSyntaxPatch(false); + await disposeSyntaxHighlighting(); + }); + await initializeSyntaxHighlighting( + { PI_GLOWUP_SYNTAX_THEME: resolve("themes/darker-modern-theme.json") }, + { preloadLanguages: ["javascript"] }, + ); + configureMarkdownSyntaxPatch(true); + const markdown = new Markdown( + `\`\`\`javascript\nconst stale = "${"note ".repeat(1_800)}";\n\`\`\``, + 0, + 0, + makeMarkdownTheme(), + ); + markdown.render(120); + markdown.setText("```javascript\nconst fresh = 1;\n```"); + markdown.render(120); + await settleSyntaxHighlights(); + const colored = markdown.render(120).join("\n"); + expect(colored).toContain("\u001b[38;2;86;156;214mconst"); + expect(colored).toContain("fresh"); + expect(colored).not.toContain("stale"); + }); + it("stops injecting a highlighter when disabled", () => { const prototype = { render(this: FakeMarkdownInstance): string[] { diff --git a/extensions/pi-glowup/test/pty/pi-cli.test.ts b/extensions/pi-glowup/test/pty/pi-cli.test.ts index 5dd448bb..938b2e34 100644 --- a/extensions/pi-glowup/test/pty/pi-cli.test.ts +++ b/extensions/pi-glowup/test/pty/pi-cli.test.ts @@ -1,12 +1,18 @@ import { visibleWidth } from "@earendil-works/pi-tui"; import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; +import { createRequire } from "node:module"; import { afterEach, describe, expect, it } from "vitest"; import { getGlowupGlobalConfigPath } from "../../src/config/load.ts"; import { PiPtyProcess, type PiProcessOptions, type PtyScreenFrame } from "./pi-process-harness.ts"; const CTRL_O = "\u000f"; +const oxfmtExecutable = join( + dirname(createRequire(import.meta.url).resolve("oxfmt/package.json")), + "bin", + "oxfmt", +); const PTY_TIMEOUT_MS = process.env.CI !== undefined ? 40_000 : 20_000; type FixtureWorkspace = { @@ -476,11 +482,7 @@ describe("actual Pi CLI in a real PTY", () => { JSON.stringify({ scriptPreview: { formatters: { - javascript: [ - process.execPath, - "-e", - "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>process.stdout.write(s.replace('; ', ';\\n')))", - ], + javascript: { kind: "javascript", command: [oxfmtExecutable] }, }, }, }), @@ -568,11 +570,7 @@ describe("actual Pi CLI in a real PTY", () => { JSON.stringify({ scriptPreview: { formatters: { - javascript: [ - process.execPath, - "-e", - "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>process.stdout.write(s.replace('; ', ';\\n')))", - ], + javascript: { kind: "javascript", command: [oxfmtExecutable] }, }, }, }), @@ -600,7 +598,7 @@ describe("actual Pi CLI in a real PTY", () => { } }, 45_000); - it("formats a standalone Node preview while the command is still running", async () => { + it("keeps original Node source while running and decorates after completion", async () => { const fixture = createFixtureWorkspace(); fixtures.push(fixture); writeFileSync( @@ -608,11 +606,7 @@ describe("actual Pi CLI in a real PTY", () => { JSON.stringify({ scriptPreview: { formatters: { - javascript: [ - process.execPath, - "-e", - "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>process.stdout.write(s.replace('; ', ';\\n')))", - ], + javascript: { kind: "javascript", command: [oxfmtExecutable] }, }, }, }), @@ -624,13 +618,23 @@ describe("actual Pi CLI in a real PTY", () => { const running = await pi.waitForFrame( (frame) => frame.text.includes("NODE_BEGIN") && - frame.rows.some((row) => row.text.includes("│ await new Promise")) && !frame.rows.some((row) => row.text.trim() === "NODE_DONE"), PTY_TIMEOUT_MS, ); expect(running.text).toContain("Node"); expect(running.text).not.toContain("STREAM_COMPLETE"); + expect(running.text).toContain('console.log("NODE_BEGIN");'); + expect(running.rows.some((row) => row.text.includes("│ await new Promise"))).toBe( + false, + ); + const completed = await pi.waitForFrame( + (frame) => + frame.text.includes("STREAM_COMPLETE") && + frame.rows.some((row) => row.text.includes("│ await new Promise")), + PTY_TIMEOUT_MS, + ); + expectTerminalInvariants(completed, [resolve("test/pty/fixtures/offline-provider.ts")]); expectTerminalInvariants(running, [resolve("test/pty/fixtures/offline-provider.ts")]); } catch (cause: unknown) { await pi.writeFailureArtifacts("running-node-formatter-preview", cause); @@ -648,11 +652,14 @@ describe("actual Pi CLI in a real PTY", () => { JSON.stringify({ scriptPreview: { formatters: { - javascript: [ - process.execPath, - "-e", - "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>setTimeout(()=>process.stdout.write(s.replace('; ', ';\\n')), 350))", - ], + javascript: { + kind: "command", + command: [ + process.execPath, + "-e", + "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>setTimeout(()=>process.stdout.write(s.replace('; ', ';\\n')), 350))", + ], + }, }, }, }), @@ -688,11 +695,14 @@ describe("actual Pi CLI in a real PTY", () => { JSON.stringify({ scriptPreview: { formatters: { - python: [ - process.execPath, - "-e", - "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>process.stdout.write(s.replace('; ', '\\n')))", - ], + python: { + kind: "command", + command: [ + process.execPath, + "-e", + "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>process.stdout.write(s.replace('; ', '\\n')))", + ], + }, }, }, }), diff --git a/extensions/pi-glowup/test/rendering/rendering.test.ts b/extensions/pi-glowup/test/rendering/rendering.test.ts index 1b04fd52..22bfd92a 100644 --- a/extensions/pi-glowup/test/rendering/rendering.test.ts +++ b/extensions/pi-glowup/test/rendering/rendering.test.ts @@ -662,7 +662,10 @@ describe("Glowup rendering helpers", () => { expect(rendered).toContain("/tmp/sample-session/env.sh"); expect(rendered).toContain("# Ensure a clean input before demo 1."); - expect(rendered).toContain('"$target"'); + expect(rendered).toContain( + '"$target"', + ); + expect(rendered).not.toContain(""); expect(rendered).toContain("-maxdepth"); expect(rendered).toContain("-name"); diff --git a/extensions/pi-glowup/test/rendering/syntax/syntax-code-component.test.ts b/extensions/pi-glowup/test/rendering/syntax/syntax-code-component.test.ts index ef0ed308..13688ce9 100644 --- a/extensions/pi-glowup/test/rendering/syntax/syntax-code-component.test.ts +++ b/extensions/pi-glowup/test/rendering/syntax/syntax-code-component.test.ts @@ -1,14 +1,205 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { stripVTControlCharacters } from "node:util"; +import type { Component } from "@earendil-works/pi-tui"; +import { createThirdPartyToolRenderer } from "../../../src/tools/renderers.ts"; +import { createPlainTheme } from "../../support/plain-theme.ts"; +import { + clearSyntaxHighlightCache, + disposeSyntaxHighlighting, + initializeSyntaxHighlighting, + highlightSyntaxCode, + highlightSyntaxCodeAsync, + observeSyntaxHighlightSource, + settleSyntaxHighlights, + syntaxHighlightCacheStats, +} from "../../../src/rendering/syntax/highlighter.ts"; import { detectStructuredOutputLanguage } from "../../../src/rendering/syntax/code-component.ts"; import { syntaxLanguageFromFile } from "../../../src/rendering/syntax/language.ts"; +import { renderGlowupOutput } from "../../../src/rendering/output.ts"; describe("syntax code component helpers", () => { + it("repaints completed Codemode long lines and retains multiline grammar state without a global large-code cache", async () => { + await initializeSyntaxHighlighting(process.env, { preloadLanguages: ["javascript"] }); + let component: Component | undefined; + const code = `const before = 1;\n/* ${"note ".repeat(1_800)}\nconst hidden = 2;\n*/ const after = "end";`; + const renderer = createThirdPartyToolRenderer("codemode"); + try { + const createComponent = (): Component => + renderer.renderCall({ code }, createPlainTheme(), { + args: { code }, + toolCallId: "long-code", + argsComplete: true, + executionStarted: true, + isPartial: false, + expanded: true, + showImages: false, + isError: false, + result: { content: [] }, + invalidate: () => { + component = createComponent(); + }, + }); + component = createComponent(); + const render = () => component?.render(12_000).join("\n") ?? ""; + const first = render(); + expect(stripVTControlCharacters(first)).toContain("const hidden = 2;"); + await vi.waitUntil(() => render().includes("\u001b[38;2;106;153;85m/* ")); + const colored = render(); + expect(colored).toContain("\u001b[38;2;106;153;85mconst hidden = 2;"); + expect(colored).toContain("\u001b[38;2;86;156;214mconst"); + expect( + stripVTControlCharacters(colored) + .split("\n") + .slice(1) + .map((line) => line.replace(/^ │ /u, "")) + .join("\n"), + ).toBe(code); + + expect(syntaxHighlightCacheStats().entries).toBe(0); + } finally { + await disposeSyntaxHighlighting(); + } + }); + + it("publishes oversized colors to reconstructed output without repeated completion", async () => { + await initializeSyntaxHighlighting({}, { preloadLanguages: ["javascript"] }); + const code = `const entries=[${"1,".repeat(18_000)}1];`; + let invalidations = 0; + const invalidate = () => { + invalidations += 1; + }; + + const render = () => + renderGlowupOutput(createPlainTheme(), code, { + expanded: true, + dimContent: false, + prefixFirst: "", + prefixRest: "", + syntax: { language: "javascript", invalidate }, + }).render(40_000); + try { + expect(render()).toEqual([code]); + await settleSyntaxHighlights(); + const colored = render(); + expect(colored.join("\n")).toContain("\u001b[38;2;86;156;214mconst"); + expect(colored.map(stripVTControlCharacters)).toEqual([code]); + expect(Buffer.byteLength(colored.join("\n"))).toBeGreaterThan(512 * 1024); + expect(render()).toEqual(colored); + await settleSyntaxHighlights(); + expect(render()).toEqual(colored); + expect(invalidations).toBe(1); + expect(syntaxHighlightCacheStats()).toEqual({ entries: 0, bytes: 0 }); + } finally { + await disposeSyntaxHighlighting(); + } + }); + + it("retains active colors across reusable-cache eviction and rejects obsolete ownership", async () => { + await initializeSyntaxHighlighting({}, { preloadLanguages: ["javascript"] }); + const code = 'const active = "retained";'; + let invalidations = 0; + const invalidate = () => { + invalidations += 1; + }; + + const current = observeSyntaxHighlightSource(code, invalidate); + try { + const colored = await highlightSyntaxCodeAsync(code, "javascript", { invalidate }); + + for (let index = 0; index < 101; index += 1) { + await highlightSyntaxCodeAsync(`const entry = ${index};`, "javascript"); + } + + expect(highlightSyntaxCode(code, "javascript", { invalidate })).toEqual(colored); + await settleSyntaxHighlights(); + expect(invalidations).toBe(1); + expect(syntaxHighlightCacheStats().entries).toBe(100); + expect(syntaxHighlightCacheStats().bytes).toBeLessThanOrEqual(512 * 1024); + + const obsolete = highlightSyntaxCodeAsync( + `/* ${"old ".repeat(3_000)} */`, + "javascript", + { + invalidate, + }, + ); + const replacement = 'const replacement = "fresh";'; + observeSyntaxHighlightSource(replacement, invalidate); + expect(current()).toBe(false); + await obsolete; + const fresh = await highlightSyntaxCodeAsync(replacement, "javascript", { invalidate }); + expect(fresh.map(stripVTControlCharacters)).toEqual([replacement]); + expect(invalidations).toBe(2); + clearSyntaxHighlightCache(); + expect(highlightSyntaxCode(replacement, "javascript", { invalidate })).toEqual([ + replacement, + ]); + await disposeSyntaxHighlighting(); + await settleSyntaxHighlights(); + expect(highlightSyntaxCode(code, "javascript", { invalidate })).toEqual([code]); + expect(invalidations).toBe(2); + } finally { + await disposeSyntaxHighlighting(); + } + }); + + it("adopts warm cache colors on the owner's first render before reusable eviction", async () => { + await initializeSyntaxHighlighting({}, { preloadLanguages: ["javascript"] }); + const code = 'const cached = "visible";'; + let invalidations = 0; + const invalidate = () => { + invalidations += 1; + }; + + try { + const colored = await highlightSyntaxCodeAsync(code, "javascript"); + expect(highlightSyntaxCode(code, "javascript", { invalidate })).toEqual(colored); + + for (let index = 0; index < 101; index += 1) { + await highlightSyntaxCodeAsync(`const entry = ${index};`, "javascript"); + } + + expect(highlightSyntaxCode(code, "javascript", { invalidate })).toEqual(colored); + await settleSyntaxHighlights(); + expect(invalidations).toBe(0); + } finally { + await disposeSyntaxHighlighting(); + } + }); + + it("cancels worker tokenization without publishing or retaining aborted source", async () => { + await initializeSyntaxHighlighting(process.env, { preloadLanguages: ["markdown"] }); + clearSyntaxHighlightCache(); + const controller = new AbortController(); + const code = `${"[".repeat(900)}${"x".repeat(900)}${"]".repeat(900)}`; + try { + const result = highlightSyntaxCodeAsync(code, "markdown", { + signal: controller.signal, + }); + controller.abort(); + expect(await result).toEqual([code]); + expect(syntaxHighlightCacheStats().entries).toBe(0); + } finally { + await disposeSyntaxHighlighting(); + } + }); + it("detects small structured output languages", () => { expect(detectStructuredOutputLanguage('{"ok":true}')).toBe("json"); expect(detectStructuredOutputLanguage("")).toBe("xml"); expect(detectStructuredOutputLanguage("git status --short")).toBe("bash"); }); + it.each([ + "rg: /work/sample: IO error", + "git: unknown command", + "npm ERR! unavailable", + "npm ERR! unavailable", + "node:internal/error", + ])("keeps command diagnostics plain: %s", (text) => { + expect(detectStructuredOutputLanguage(text)).toBeUndefined(); + }); + it("skips structured detection for huge outputs", () => { const hugeJson = `{${'"value":'.padStart(65 * 1024, " ")}true}`; diff --git a/extensions/pi-glowup/test/rendering/syntax/syntax-highlighter-lifecycle.test.ts b/extensions/pi-glowup/test/rendering/syntax/syntax-highlighter-lifecycle.test.ts index 62d85544..995223da 100644 --- a/extensions/pi-glowup/test/rendering/syntax/syntax-highlighter-lifecycle.test.ts +++ b/extensions/pi-glowup/test/rendering/syntax/syntax-highlighter-lifecycle.test.ts @@ -17,6 +17,8 @@ import { getLoadedSyntaxHighlighterForLanguage, getSyntaxHighlighterForLanguage, highlightSyntaxCode, + highlightSyntaxCodeAsync, + settleSyntaxHighlights, initializeSyntaxHighlighting, isSyntaxHighlightingReady, loadSyntaxLanguageIfReady, @@ -737,7 +739,7 @@ describe("syntax highlighter lifecycle", () => { "const value: string = 'ok';", "TS", ].join("\n"); - let component: Component = renderScriptCall( + const component: Component = renderScriptCall( plainTheme, { label: "Bash", language: "bash", code }, { @@ -745,25 +747,15 @@ describe("syntax highlighter lifecycle", () => { expanded: false, invalidate() { invalidations += 1; + component.invalidate(); }, }, ); expect(component.render(120).join("\n")).not.toContain("\u001b["); await waitForCondition(() => invalidations > 0); - - component = renderScriptCall( - plainTheme, - { label: "Bash", language: "bash", code }, - { - state: "success", - expanded: false, - invalidate() { - invalidations += 1; - }, - }, - ); - + component.render(120); + await settleSyntaxHighlights(); const rendered = component.render(120).join("\n"); expect(rendered).toContain("\u001b["); @@ -776,7 +768,10 @@ describe("syntax highlighter lifecycle", () => { it("disposes and resets highlighter state between sessions", async () => { await initializeSyntaxHighlighting(); expect(isSyntaxHighlightingReady()).toBe(true); - expect(highlightSyntaxCode("import sys", "python").join("\n")).toContain("\u001b["); + expect((await highlightSyntaxCodeAsync("import sys", "python")).join("\n")).toContain( + "\u001b[", + ); + expect(syntaxHighlightCacheStats().entries).toBeGreaterThan(0); await disposeSyntaxHighlighting(); @@ -795,11 +790,11 @@ describe("syntax highlighter lifecycle", () => { await initializeSyntaxHighlighting(); clearSyntaxHighlightCache(); - highlightSyntaxCode("streamed = 1", "python", { cache: false }); + await highlightSyntaxCodeAsync("streamed = 1", "python", { cache: false }); expect(syntaxHighlightCacheStats()).toEqual({ entries: 0, bytes: 0 }); - highlightSyntaxCode("final_value = 1", "python"); + await highlightSyntaxCodeAsync("final_value = 1", "python"); expect(syntaxHighlightCacheStats().entries).toBe(1); expect(syntaxHighlightCacheStats().bytes).toBeGreaterThan(0); diff --git a/extensions/pi-glowup/test/rendering/syntax/syntax.test.ts b/extensions/pi-glowup/test/rendering/syntax/syntax.test.ts index e5ef6854..5a36a567 100644 --- a/extensions/pi-glowup/test/rendering/syntax/syntax.test.ts +++ b/extensions/pi-glowup/test/rendering/syntax/syntax.test.ts @@ -1,4 +1,5 @@ import { join } from "node:path"; +import { stripVTControlCharacters } from "node:util"; import { Markdown, type MarkdownTheme } from "@earendil-works/pi-tui"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { Theme, type ThemeColor } from "@earendil-works/pi-coding-agent"; @@ -18,6 +19,8 @@ import { disposeSyntaxHighlighting, configureSyntaxBracketPairColoring, highlightSyntaxCode, + highlightSyntaxCodeAsync, + settleSyntaxHighlights, initializeSyntaxHighlighting, } from "../../../src/rendering/syntax/highlighter.ts"; import { normalizeSyntaxLanguage } from "../../../src/rendering/syntax/language.ts"; @@ -61,6 +64,7 @@ const piTheme = new Theme( syntaxFunction: "#dcdcaa", syntaxOperator: "#d4d4d4", syntaxString: "#ce9178", + syntaxVariable: "#9cdcfe", toolOutput: "#d4d4d4", success: "#89d185", thinkingXhigh: "#777777", @@ -101,6 +105,18 @@ const renderContext = { }; const bundledThemePath = join(process.cwd(), "themes", "darker-modern-theme.json"); +async function renderReadyDiff( + sections: Parameters[1], + expanded: boolean, + width: number, +): Promise { + const component = renderGlowupDiff(plainTheme, sections, expanded); + component.render(width); + await settleSyntaxHighlights(); + + return component.render(width).join("\n"); +} + describe("central syntax highlighting", () => { beforeAll(async () => { await disposeSyntaxHighlighting(); @@ -140,35 +156,92 @@ describe("central syntax highlighting", () => { }); }); - it("highlights TypeScript with the VS Code theme colors", () => { - const lines = highlightSyntaxCode('const value = "ok";', "typescript"); + it("highlights TypeScript with the VS Code theme colors", async () => { + const lines = await highlightSyntaxCodeAsync('const value = "ok";', "typescript"); expect(lines.join("\n")).toContain(TYPESCRIPT_KEYWORD_COLOR); expect(lines.join("\n")).toContain("const"); }); - it("highlights accepted one-line scripts longer than 1000 characters", () => { + it("preserves grammar state through an oversized multiline comment", async () => { + const code = `const before = "safe";\n/* ${"x".repeat(9_000)}\nconst inside = "not code";\n*/\nconst after = 1;`; + const lines = await highlightSyntaxCodeAsync(code, "javascript", { cache: false }); + expect(lines[0]).toContain(TYPESCRIPT_KEYWORD_COLOR); + expect(lines[1]).toContain(ANSI_ESCAPE); + expect(lines[2]).not.toContain(TYPESCRIPT_KEYWORD_COLOR); + expect(lines[4]).toContain(TYPESCRIPT_KEYWORD_COLOR); + expect(stripVTControlCharacters(lines.join("\n"))).toBe(code); + }); + + it("highlights accepted one-line scripts longer than 1000 characters", async () => { const code = `const values = [${Array.from({ length: 140 }, (_value, index) => `"value-${index}"`).join(",")}];`; expect(code.length).toBeGreaterThan(1_000); expect(code.length).toBeLessThanOrEqual(2_000); - expect(highlightSyntaxCode(code, "javascript").join("\n")).toContain( + expect((await highlightSyntaxCodeAsync(code, "javascript")).join("\n")).toContain( TYPESCRIPT_KEYWORD_COLOR, ); }); - it("does not reset surrounding backgrounds after highlighted lines", () => { - const rendered = highlightSyntaxCode('const value = "ok";', "typescript").join("\n"); + it("highlights long logical lines verbatim within the block budget", async () => { + const code = `const values = [${Array.from({ length: 230 }, (_value, index) => `"item-${index}"`).join(",")}];`; + const highlighted = (await highlightSyntaxCodeAsync(code, "javascript")).join("\n"); + expect(highlighted).toContain(TYPESCRIPT_KEYWORD_COLOR); + expect(stripVTControlCharacters(highlighted)).toBe(code); + const excessive = `const value = "${"x".repeat(16_000)}";`; + const large = (await highlightSyntaxCodeAsync(excessive, "javascript")).join("\n"); + expect(large).toContain(TYPESCRIPT_KEYWORD_COLOR); + expect(stripVTControlCharacters(large)).toBe(excessive); + }); + + it("distinguishes Bash variables from literal quoted text and redirect operands", () => { + const rendered = renderScriptCall( + piTheme, + { + label: "Bash", + language: "bash", + code: `result=$?; printf '%s\\n' "literal $result"; rg word 2>/dev/null; > before.log git status`, + }, + { state: "success", expanded: true }, + ) + .render(240) + .join("\n"); + expect(rendered).toContain("38;2;156;220;254mresult"); + expect(rendered).toContain("38;2;156;220;254m$?"); + expect(rendered).toContain("38;2;156;220;254m$result"); + expect(rendered).toContain(`${STRING_RGB_CODE}m'%s\\n'`); + expect(rendered).toContain("38;2;255;255;255m/dev/null"); + expect(rendered).toContain("38;2;255;255;255mbefore.log"); + expect(rendered).toContain("38;2;220;220;170mgit"); + const command = `NAME="literal $value" printf 'literal $value' "\\$value" "\${value:-fallback}" 3>log.txt 4>&1`; + const mixed = renderScriptCall( + piTheme, + { label: "Bash", language: "bash", code: command }, + { state: "success", expanded: true }, + ) + .render(240) + .join("\n"); + expect(stripVTControlCharacters(mixed)).toBe(`• Bash ${command}`); + expect(mixed).toContain(`${STRING_RGB_CODE}m'literal $value'`); + expect(mixed).toContain("38;2;156;220;254m$\u007bvalue:-fallback}"); + }); + + it("does not reset surrounding backgrounds after highlighted lines", async () => { + const rendered = (await highlightSyntaxCodeAsync('const value = "ok";', "typescript")).join( + "\n", + ); expect(rendered).not.toContain("\u001b[0m"); expect(rendered).not.toContain("\u001b[49m"); expect(rendered).not.toContain("\u001b[48;"); }); - it("colors Python import identifiers like VS Code semantic highlighting", () => { - const rendered = highlightSyntaxCode( - "import sys\nfrom collections.abc import Callable", - "python", + it("colors Python import identifiers like VS Code semantic highlighting", async () => { + const rendered = ( + await highlightSyntaxCodeAsync( + "import sys\nfrom collections.abc import Callable", + "python", + ) ).join("\n"); expect(rendered).toContain(`${TYPE_RGB_CODE}msys`); @@ -176,10 +249,12 @@ describe("central syntax highlighting", () => { expect(rendered).toContain(`${TYPE_RGB_CODE}mCallable`); }); - it("colors Python neutral identifiers as variables, constants, and calls", () => { - const rendered = highlightSyntaxCode( - "if IGNORE_BREAKS:\n handler(line)\n value = get_terminal_size().columns", - "python", + it("colors Python neutral identifiers as variables, constants, and calls", async () => { + const rendered = ( + await highlightSyntaxCodeAsync( + "if IGNORE_BREAKS:\n handler(line)\n value = get_terminal_size().columns", + "python", + ) ).join("\n"); expect(rendered).toContain(`${TYPE_RGB_CODE}mIGNORE_BREAKS`); @@ -189,10 +264,9 @@ describe("central syntax highlighting", () => { expect(rendered).toContain(`${FUNCTION_RGB_CODE}mget_terminal_size`); }); - it("adds VS Code bracket pair colors without recoloring strings", () => { - const rendered = highlightSyntaxCode( - 'const value = make([]Item, "(text)");', - "typescript", + it("adds VS Code bracket pair colors without recoloring strings", async () => { + const rendered = ( + await highlightSyntaxCodeAsync('const value = make([]Item, "(text)");', "typescript") ).join("\n"); expect(rendered).toContain(BRACKET_PAIR_1_RGB_CODE); @@ -200,12 +274,14 @@ describe("central syntax highlighting", () => { expect(rendered).toContain(`${STRING_RGB_CODE}m"(text)"`); }); - it("preserves syntax-theme bracket colors when pair coloring is disabled", () => { + it("preserves syntax-theme bracket colors when pair coloring is disabled", async () => { configureSyntaxBracketPairColoring(false); try { - const rendered = highlightSyntaxCode( - 'const value = make([]Item, "(text)");', - "typescript", + const rendered = ( + await highlightSyntaxCodeAsync( + 'const value = make([]Item, "(text)");', + "typescript", + ) ).join("\n"); expect(rendered).not.toContain(BRACKET_PAIR_1_RGB_CODE); @@ -230,7 +306,8 @@ describe("central syntax highlighting", () => { expect(normalizeSyntaxLanguage("golang")).toBe("go"); }); - it("injects the central highlighter into Markdown code fences", () => { + it("injects the central highlighter into Markdown code fences", async () => { + await highlightSyntaxCodeAsync("const value = 1;", "typescript"); installMarkdownSyntaxPatch(); const rendered = new Markdown("```ts\nconst value = 1;\n```", 0, 0, markdownTheme) .render(100) @@ -253,7 +330,8 @@ describe("central syntax highlighting", () => { expect(rendered).toContain("const value = 1;"); }); - it("highlights bash command and executable script previews", () => { + it("highlights bash command and executable script previews", async () => { + await highlightSyntaxCodeAsync("const value = 1;", "javascript"); const rendered = renderScriptCall( plainTheme, { label: "Node", language: "javascript", code: "const value = 1;" }, @@ -292,7 +370,7 @@ describe("central syntax highlighting", () => { expect(rendered).not.toContain(`${STRING_RGB_CODE}minstall`); }); - it("preserves multiline TypeScript comment highlighting in script previews", () => { + it("preserves multiline TypeScript comment highlighting in script previews", async () => { const code = [ "/**", " * Expands the collapsed paste marker currently under the editor cursor.", @@ -301,7 +379,7 @@ describe("central syntax highlighting", () => { " return true;", "}", ].join("\n"); - const expectedCommentLine = highlightSyntaxCode(code, "typescript")[1]; + const expectedCommentLine = (await highlightSyntaxCodeAsync(code, "typescript"))[1]; const renderedCommentLine = renderScriptCall( plainTheme, { label: "TypeScript", language: "typescript", code }, @@ -314,7 +392,7 @@ describe("central syntax highlighting", () => { expect(renderedCommentLine).toContain(expectedCommentLine); }); - it("preserves multiline TypeScript highlighting inside bash heredocs", () => { + it("preserves multiline TypeScript highlighting inside bash heredocs", async () => { const code = [ "/**", " * Expands the collapsed paste marker currently under the editor cursor.", @@ -327,7 +405,7 @@ describe("central syntax highlighting", () => { code, "TS", ].join("\n"); - const expectedCommentLine = highlightSyntaxCode(code, "typescript")[1]; + const expectedCommentLine = (await highlightSyntaxCodeAsync(code, "typescript"))[1]; const rendered = renderScriptCall( plainTheme, { label: "Bash", language: "bash", code: command }, @@ -340,7 +418,11 @@ describe("central syntax highlighting", () => { expect(renderedCommentLine).toContain(expectedCommentLine); }); - it("highlights embedded Python heredocs inside bash previews as Python", () => { + it("highlights embedded Python heredocs inside bash previews as Python", async () => { + await highlightSyntaxCodeAsync( + "import json, sys\npath = sys.argv[1]\nprint(json.load(open(path)).get('$schema'))", + "python", + ); const command = [ 'for f in /tmp/*.json; do [ -f "$f" ] || continue;', "printf '%s: ' \"$f\"; python3 - <<'PY' \"$f\"", @@ -364,7 +446,11 @@ describe("central syntax highlighting", () => { expect(rendered).not.toContain(`${STRING_RGB_CODE}mimport json, sys`); }); - it("highlights single-quoted inline Python inside a composed Bash pipeline", () => { + it("highlights single-quoted inline Python inside a composed Bash pipeline", async () => { + await highlightSyntaxCodeAsync( + 'import sys\nfor line in sys.stdin:\n value, name = line.split()\n print(f"{int(value):04d} {name}")\n', + "python", + ); const command = [ "printf '%s\\n' '8 read' '13 patch' | python3 -c 'import sys", "for line in sys.stdin:", @@ -414,7 +500,8 @@ describe("central syntax highlighting", () => { expect(rendered.join("\n")).not.toContain(`\u001b[${TOML_INVALID_RGB_CODE}m`); }); - it("highlights code output previews when a path is known", () => { + it("highlights code output previews when a path is known", async () => { + await highlightSyntaxCodeAsync("const value = 1;", "typescript"); const rendered = renderGlowupOutput(plainTheme, "const value = 1;", { expanded: false, syntax: { path: "src/example.ts" }, @@ -426,7 +513,7 @@ describe("central syntax highlighting", () => { expect(rendered).toContain("const"); }); - it("preserves multiline TypeScript comment highlighting in code output previews", () => { + it("preserves multiline TypeScript comment highlighting in code output previews", async () => { const code = [ "/**", " * Expands the collapsed paste marker currently under the editor cursor.", @@ -435,7 +522,7 @@ describe("central syntax highlighting", () => { " return true;", "}", ].join("\n"); - const expectedCommentLine = highlightSyntaxCode(code, "typescript")[1]; + const expectedCommentLine = (await highlightSyntaxCodeAsync(code, "typescript"))[1]; const renderedCommentLine = renderGlowupOutput(plainTheme, code, { expanded: false, maxPreviewLines: 8, @@ -448,7 +535,8 @@ describe("central syntax highlighting", () => { expect(renderedCommentLine).toContain(expectedCommentLine); }); - it("highlights partial write previews while streaming", () => { + it("highlights partial write previews while streaming", async () => { + await highlightSyntaxCodeAsync("const value = 1;\n", "typescript"); const rendered = renderWriteCallPreview( { path: "src/example.ts", content: "const value = 1;\n" }, plainTheme, @@ -461,7 +549,8 @@ describe("central syntax highlighting", () => { expect(rendered).toContain("const"); }); - it("highlights third-party JSON fallback previews", () => { + it("highlights third-party JSON fallback previews", async () => { + await highlightSyntaxCodeAsync('{"ok": true}', "json"); const renderer = createThirdPartyToolRenderer("custom_tool"); const rendered = renderer .renderResult( @@ -477,7 +566,7 @@ describe("central syntax highlighting", () => { expect(rendered).toContain("ok"); }); - it("highlights fallback diff previews when a path is known", () => { + it("highlights fallback diff previews when a path is known", async () => { configureRenderingAppearance({ diffBackgroundStyle: "two-tone", diffLineNumberStyle: "dual", @@ -490,24 +579,21 @@ describe("central syntax highlighting", () => { instructionPathColor: null, dimUnchangedDiffText: false, }); - const rendered = renderGlowupDiff( - plainTheme, + const rendered = await renderReadyDiff( parseDiffSections( " 7 import pytest\n+9 def test_value():\n-9 def old_value():", "tests/example.py", ), false, - ) - .render(120) - .join("\n"); + 120, + ); expect(rendered).toContain("def"); expect(rendered).toContain("38;2"); }); - it("uses surrounding TOML diff context for inline table braces", () => { - const rendered = renderGlowupDiff( - plainTheme, + it("uses surrounding TOML diff context for inline table braces", async () => { + const rendered = await renderReadyDiff( parseDiffSections( [ " 1 [[package]]", @@ -519,17 +605,15 @@ describe("central syntax highlighting", () => { "config/services/sample.toml", ), true, - ) - .render(180) - .join("\n"); + 180, + ); expect(rendered).toContain(BRACKET_PAIR_2_RGB_CODE); expect(rendered).not.toContain(TOML_INVALID_RGB_CODE); }); - it("does not highlight TOML after diff ellipses as invalid", () => { - const rendered = renderGlowupDiff( - plainTheme, + it("does not highlight TOML after diff ellipses as invalid", async () => { + const rendered = await renderReadyDiff( parseDiffSections( [ " 1 [[package]]", @@ -554,9 +638,8 @@ describe("central syntax highlighting", () => { "config/services/sample.toml", ), false, - ) - .render(180) - .join("\n"); + 180, + ); expect(rendered).toContain(BRACKET_PAIR_2_RGB_CODE); expect(rendered).not.toContain(TOML_INVALID_RGB_CODE); diff --git a/extensions/pi-glowup/test/tools/built-in/bash/bash-analysis.test.ts b/extensions/pi-glowup/test/tools/built-in/bash/bash-analysis.test.ts index aa09d07f..8b31c89f 100644 --- a/extensions/pi-glowup/test/tools/built-in/bash/bash-analysis.test.ts +++ b/extensions/pi-glowup/test/tools/built-in/bash/bash-analysis.test.ts @@ -1,3 +1,4 @@ +import { spawnSync } from "node:child_process"; import { describe, expect, it } from "vitest"; import { @@ -181,6 +182,41 @@ describe("bash command analysis", () => { ).toBe(`git add src/one.ts &&\ngit commit -m "fix: handle imports for scripts"`); }); + it.each([ + ["else # else\n", "else # else\n"], + ["else # the else branch\n", "else # the else branch\n"], + ["else\n# else\n", "else\n# else\n"], + ])("keeps branch comments lexical: %s", (branch, preserved) => { + const source = `if check; then first; ${branch}second; fi`; + const displayed = reflowBashCommand(source); + expect(displayed).toBe(`if check; then\n first;\n${preserved}second;\nfi`); + expect(spawnSync("bash", ["--noprofile", "--norc", "-n"], { input: source }).status).toBe( + 0, + ); + + expect( + spawnSync("bash", ["--noprofile", "--norc", "-n"], { input: displayed }).status, + ).toBe(0); + }); + + it.each([ + "if check; then printf '%s' 'else fi'; elif other; then if nested; then first; else # else\nsecond; fi; else third; fi", + "for item in done; do if check; then first; else # else done\nsecond; fi; # done\ndone", + "while check; do case value in first) printf '%s' 'esac done';; *) if other; then first; else # else esac\nsecond; fi;; esac; done", + "if check; then cat <<'DATA'\nelse fi done esac\n# else\nDATA\nelse # else\nsecond; fi", + ])("preserves nested structure and keyword-bearing literals: %s", (source) => { + const displayed = reflowBashCommand(source); + expect(displayed).toBeDefined(); + expect(displayed).toContain("else # else"); + expect(spawnSync("bash", ["--noprofile", "--norc", "-n"], { input: source }).status).toBe( + 0, + ); + + expect( + spawnSync("bash", ["--noprofile", "--norc", "-n"], { input: displayed }).status, + ).toBe(0); + }); + it("keeps leading operators selectable", () => { expect(reflowBashCommand(`one && two && three`, "leading")).toBe(`one\n&& two\n&& three`); expect(reflowBashCommand(`case "$x" in a) one && two;; esac`, "leading")).toContain( @@ -194,6 +230,33 @@ describe("bash command analysis", () => { expect(analyzeBashCommand(heredoc).pureScript).toBeUndefined(); }); + it("splits shell statements around heredocs without changing their content", () => { + const command = "first; second\ncat <<'DATA'\n literal; data \n\nDATA\nthird; fourth"; + expect(reflowBashCommand(command)).toBe( + "first;\nsecond\ncat <<'DATA'\n literal; data \n\nDATA\nthird;\nfourth", + ); + + expect( + reflowBashCommand("first; cat <<'DATA'; attached\nbody; text\nDATA\nlast; final"), + ).toBe("first;\ncat <<'DATA'; attached\nbody; text\nDATA\nlast;\nfinal"); + + expect( + reflowBashCommand( + "first; cat <<-ONE <<'TWO'\n\tdata; first\n\tONE\n data; second\nTWO\nlast; final", + ), + ).toBe( + "first;\ncat <<-ONE <<'TWO'\n\tdata; first\n\tONE\n data; second\nTWO\nlast;\nfinal", + ); + + expect(reflowBashCommand('printf "first\n second; still text"; final')).toBe( + 'printf "first\n second; still text";\nfinal', + ); + + expect( + reflowBashCommand("if check; then cat <<'DATA'\nfi else done\nDATA\nelse other; fi"), + ).toBe("if check; then\n cat <<'DATA'\nfi else done\nDATA\nelse\n other;\nfi"); + }); + it("falls back without reflowing malformed syntax", () => { expect(reflowBashCommand(`echo "unfinished && just test`)).toBeUndefined(); }); diff --git a/extensions/pi-glowup/test/tools/built-in/bash/bash-call-rendering.test.ts b/extensions/pi-glowup/test/tools/built-in/bash/bash-call-rendering.test.ts index 819d6f64..499f3288 100644 --- a/extensions/pi-glowup/test/tools/built-in/bash/bash-call-rendering.test.ts +++ b/extensions/pi-glowup/test/tools/built-in/bash/bash-call-rendering.test.ts @@ -64,6 +64,27 @@ describe("bash command rendering", () => { expect(rendered).toContain("│ && two"); }); + it.each(["auto", "always"] as const)("retains branch comments in %s layout", (shellLayout) => { + const command = "if check; then first; else # the else branch\nsecond; fi"; + const rows = renderBashCommandCall(theme, command, { + state: "success", + expanded: true, + maxCodePreviewLines: 8, + showPrologueOmission: false, + headerLayout: "auto", + shellLayout, + shellOperatorPosition: "trailing", + }).render(120); + expect(rows).toEqual([ + "• Bash", + " │ if check; then", + " │ first;", + " │ else # the else branch", + " │ second;", + " │ fi", + ]); + }); + it("keeps pipelines and fallbacks on their step line when they fit", () => { const command = `pwd && opencode debug --help | head -80 ` + @@ -166,6 +187,28 @@ describe("bash command rendering", () => { expect(rendered).not.toContain("• Python"); }); + it("reveals nine fitting terminal rows but keeps taller script content capped", () => { + const command = `python -c '${["x=1", "x=2", "x=3", "x=4", "x=5", "x=6", "x=7", "print(12345678)"].join("\n")}'`; + const component = renderBashCommandCall(theme, command, { + state: "success", + expanded: false, + maxCodePreviewLines: 8, + showPrologueOmission: false, + headerLayout: "block", + shellLayout: "auto", + shellOperatorPosition: "trailing", + }); + const fitting = component.render(18); + expect(fitting).toHaveLength(10); + expect(fitting.at(-1)).toContain("78)"); + expect(fitting.join("\n")).not.toContain("…"); + const taller = component.render(12); + expect(taller.join("\n")).toContain("…"); + expect(taller.join("\n")).not.toContain("78)"); + expect(taller.every((line) => visibleWidth(line) <= 12)).toBe(true); + expect(component.render(18)).toEqual(fitting); + }); + it("renders clean standalone interpreter calls as language previews", () => { const rendered = render(`python -c "import os; print(os.getcwd())"`, { width: 80, diff --git a/extensions/pi-glowup/test/tools/built-in/bash/script-formatters.test.ts b/extensions/pi-glowup/test/tools/built-in/bash/script-formatters.test.ts index e249c3ea..e91456c4 100644 --- a/extensions/pi-glowup/test/tools/built-in/bash/script-formatters.test.ts +++ b/extensions/pi-glowup/test/tools/built-in/bash/script-formatters.test.ts @@ -1,6 +1,8 @@ import { existsSync, readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; +import { createRequire } from "node:module"; +import type { ScriptFormatterProfile } from "../../../../src/settings-input.ts"; import { describe, expect, it, vi } from "vitest"; import { createCommandScriptFormatter, @@ -11,6 +13,76 @@ import { } from "../../../../src/tools/built-in/bash/formatter.ts"; describe("script formatter settings", () => { + it("compacts JavaScript objects with Oxfmt at the requested display width", async () => { + const executable = join( + dirname(createRequire(import.meta.url).resolve("oxfmt/package.json")), + "bin", + "oxfmt", + ); + const formatter = createCommandScriptFormatter( + parseScriptFormatterCommandsValue({ + javascript: { kind: "javascript", command: [executable] }, + }), + ); + const invocation = { + label: "Node", + language: "javascript", + code: "const entry = {\n alpha: 1,\n beta: 2,\n};", + }; + const wide = await formatScriptInvocation(invocation, formatter, { targetWidth: 80 }); + expect(wide.code).toBe("const entry = { alpha: 1, beta: 2 };"); + const narrow = await formatScriptInvocation(invocation, formatter, { targetWidth: 20 }); + expect(narrow.code).toBe("const entry = {\n alpha: 1,\n beta: 2,\n};"); + expect(invocation.code).toBe("const entry = {\n alpha: 1,\n beta: 2,\n};"); + }); + + it("selects the TypeScript parser for compact typed scripts", async () => { + const executable = join( + dirname(createRequire(import.meta.url).resolve("oxfmt/package.json")), + "bin", + "oxfmt", + ); + const formatter = createCommandScriptFormatter( + parseScriptFormatterCommandsValue({ + typescript: { kind: "typescript", command: [executable] }, + }), + ); + const formatted = await formatScriptInvocation( + { + label: "TypeScript", + language: "typescript", + code: "const entries:number[]=[1,2,3];", + }, + formatter, + { targetWidth: 80 }, + ); + expect(formatted.code).toBe("const entries: number[] = [1, 2, 3];"); + }); + + it("reports an unavailable configured formatter once without exposing source", async () => { + const warnings: string[] = []; + const formatter = createCommandScriptFormatter( + parseScriptFormatterCommandsValue({ + javascript: { kind: "javascript", command: ["/missing/display-formatter"] }, + }), + (message) => { + warnings.push(message); + }, + ); + const invocation = { + label: "Node", + language: "javascript", + code: "const privateValue = 1;", + }; + await expect( + formatScriptInvocation(invocation, formatter, { targetWidth: 80 }), + ).resolves.toEqual(invocation); + await expect( + formatScriptInvocation(invocation, formatter, { targetWidth: 40 }), + ).resolves.toEqual(invocation); + expect(warnings).toEqual(["[pi-glowup] javascript display formatter executable not found"]); + }); + it("leaves formatter commands empty by default", () => { expect(parseScriptFormatterCommands(undefined).size).toBe(0); expect(parseScriptFormatterCommands("").size).toBe(0); @@ -37,10 +109,10 @@ describe("script formatter settings", () => { const warnings: string[] = []; const commands = parseScriptFormatterCommandsValue( { - python: ["black", "-"], - javascript: [], - typescript: [""], - ruby: ["ruby\0"], + python: { kind: "command", command: ["black", "-"] }, + javascript: { kind: "command", command: [] }, + typescript: { kind: "command", command: [""] }, + ruby: { kind: "command", command: ["ruby\0"] }, }, { source: "config.scriptPreview.formatters", @@ -50,7 +122,7 @@ describe("script formatter settings", () => { }, ); - expect(commands.get("python")).toEqual(["black", "-"]); + expect(commands.get("python")).toEqual({ kind: "command", command: ["black", "-"] }); expect(commands.has("javascript")).toBe(false); expect(commands.has("typescript")).toBe(false); expect(commands.has("ruby")).toBe(false); @@ -62,19 +134,25 @@ describe("script formatter settings", () => { it("parses language formatter commands from JSON", () => { const commands = parseScriptFormatterCommands( JSON.stringify({ - python: ["black", "-q", "-"], - javascript: ["prettier", "--parser", "babel"], + python: { kind: "command", command: ["black", "-q", "-"] }, + javascript: { kind: "command", command: ["prettier", "--parser", "babel"] }, }), ); - expect(commands.get("python")).toEqual(["black", "-q", "-"]); - expect(commands.get("javascript")).toEqual(["prettier", "--parser", "babel"]); + expect(commands.get("python")).toEqual({ kind: "command", command: ["black", "-q", "-"] }); + expect(commands.get("javascript")).toEqual({ + kind: "command", + command: ["prettier", "--parser", "babel"], + }); }); it("applies configured command formatters by language", async () => { const commands = parseScriptFormatterCommands( JSON.stringify({ - python: [process.execPath, "-e", "process.stdin.pipe(process.stdout)"], + python: { + kind: "command", + command: [process.execPath, "-e", "process.stdin.pipe(process.stdout)"], + }, }), ); const formatter = createCommandScriptFormatter(commands); @@ -96,12 +174,15 @@ describe("script formatter settings", () => { it("substitutes width placeholders and passes environment variables", async () => { const commands = parseScriptFormatterCommands( JSON.stringify({ - python: [ - process.execPath, - "-e", - "process.stdout.write(process.argv[1] + ':' + process.env.TUFF_LINE_LENGTH)", - "{width}", - ], + python: { + kind: "command", + command: [ + process.execPath, + "-e", + "process.stdout.write(process.argv[1] + ':' + process.env.TUFF_LINE_LENGTH)", + "{width}", + ], + }, }), ); const formatter = createCommandScriptFormatter(commands); @@ -121,12 +202,15 @@ describe("script formatter settings", () => { try { const commands = parseScriptFormatterCommands( JSON.stringify({ - python: [ - process.execPath, - "-e", - "require('node:fs').writeFileSync(process.argv[1], 'started');process.stdout.write('FORMATTER_RAN')", - marker, - ], + python: { + kind: "command", + command: [ + process.execPath, + "-e", + "require('node:fs').writeFileSync(process.argv[1], 'started');process.stdout.write('FORMATTER_RAN')", + marker, + ], + }, }), ); const formatter = createCommandScriptFormatter(commands); @@ -143,14 +227,17 @@ describe("script formatter settings", () => { it("reuses successful formatting for identical language source", async () => { const formatter = createCommandScriptFormatter( - new Map([ + new Map([ [ "python", - [ - process.execPath, - "-e", - "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>process.stdout.write(process.pid+':'+s))", - ], + { + kind: "command", + command: [ + process.execPath, + "-e", + "let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>process.stdout.write(process.pid+':'+s))", + ], + }, ], ]), ); @@ -166,10 +253,23 @@ describe("script formatter settings", () => { it("falls back when command formatters fail or return empty output", async () => { const invocation = { label: "Python", language: "python", code: "print(1)" }; const failingFormatter = createCommandScriptFormatter( - new Map([["python", [process.execPath, "-e", "process.exit(1)"]]]), + new Map([ + [ + "python", + { kind: "command", command: [process.execPath, "-e", "process.exit(1)"] }, + ], + ]), ); const emptyFormatter = createCommandScriptFormatter( - new Map([["python", [process.execPath, "-e", "process.stdin.resume()"]]]), + new Map([ + [ + "python", + { + kind: "command", + command: [process.execPath, "-e", "process.stdin.resume()"], + }, + ], + ]), ); await expect(formatScriptInvocation(invocation, failingFormatter)).resolves.toEqual( @@ -189,7 +289,11 @@ describe("script formatter settings", () => { "process.stdin.on('data', chunk => input += chunk);", "process.stdin.on('end', () => process.stdout.write(input + ' formatted'));", ].join("\n"); - const commands = new Map([["python", [process.execPath, "-e", script]]]); + + const commands = new Map([ + ["python", { kind: "command", command: [process.execPath, "-e", script] }], + ]); + const formatter = createCommandScriptFormatter(commands); const invocation = { label: "Python", language: "python", code: "print(1)" }; const controller = new AbortController(); @@ -228,7 +332,9 @@ describe("script formatter settings", () => { ].join("\n"); const formatter = createCommandScriptFormatter( - new Map([["python", [process.execPath, "-e", delayedEcho]]]), + new Map([ + ["python", { kind: "command", command: [process.execPath, "-e", delayedEcho] }], + ]), ); const first = { label: "Python", language: "python", code: "print(1)" }; const second = { label: "Python", language: "python", code: "print(2)" }; @@ -294,7 +400,9 @@ describe("script formatter settings", () => { "});", ].join("\n"); const formatter = createCommandScriptFormatter( - new Map([["python", [process.execPath, "-e", script]]]), + new Map([ + ["python", { kind: "command", command: [process.execPath, "-e", script] }], + ]), ); const controller = new AbortController(); const invoke = async (code: string, signal?: AbortSignal) => @@ -347,7 +455,9 @@ describe("script formatter settings", () => { failure === "overflow" ? "process.stdout.write('x'.repeat(1024 * 1024 + 1));" : "", ].join("\n"); const formatter = createCommandScriptFormatter( - new Map([["python", [process.execPath, "-e", script]]]), + new Map([ + ["python", { kind: "command", command: [process.execPath, "-e", script] }], + ]), ); const controller = new AbortController(); const invocation = { label: "Python", language: "python", code: failure }; @@ -372,14 +482,17 @@ describe("script formatter settings", () => { it("accounts repeated same-key concurrent replacements without evicting retained outputs", async () => { const formatter = createCommandScriptFormatter( - new Map([ + new Map([ [ "python", - [ - process.execPath, - "-e", - "process.stdin.resume();process.stdin.on('end',()=>process.stdout.write(process.pid+':'+'é'.repeat(350000)))", - ], + { + kind: "command", + command: [ + process.execPath, + "-e", + "process.stdin.resume();process.stdin.on('end',()=>process.stdout.write(process.pid+':'+'é'.repeat(350000)))", + ], + }, ], ]), ); diff --git a/extensions/pi-glowup/test/tools/built-in/bash/script-preview-events.test.ts b/extensions/pi-glowup/test/tools/built-in/bash/script-preview-events.test.ts index 859e71a9..4600bc59 100644 --- a/extensions/pi-glowup/test/tools/built-in/bash/script-preview-events.test.ts +++ b/extensions/pi-glowup/test/tools/built-in/bash/script-preview-events.test.ts @@ -1,21 +1,207 @@ -import { describe, expect, it } from "vitest"; -import type { ScriptBlockFormatter } from "../../../../src/tools/built-in/bash/formatter.ts"; +import { describe, expect, it, vi } from "vitest"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { stripVTControlCharacters } from "node:util"; +import { Theme } from "@earendil-works/pi-coding-agent"; +import { format, type FormatConfig } from "oxfmt"; +import { TEST_THEME_BACKGROUND_COLORS, TEST_THEME_COLORS } from "../../../support/theme-colors.ts"; +import { + createCommandScriptFormatter, + getScriptFormatterWorkload, + parseScriptFormatterCommandsValue, + type ScriptBlockFormatter, + type ScriptBlockFormatterInput, + type ScriptBlockFormatterOptions, +} from "../../../../src/tools/built-in/bash/formatter.ts"; import { createNativeBashFeature } from "../../../../src/tools/built-in/bash/preview.ts"; import { rememberRawScriptPreview, scheduleFormattedScriptPreview, } from "../../../../src/tools/built-in/bash/format-preview.ts"; import type { ScriptInvocation } from "../../../../src/tools/built-in/bash/invocation.ts"; +import type { BuiltInRenderContext } from "../../../../src/tools/built-in/context.ts"; class RecordingPreviewSink { readonly previews = new Map(); + get(toolCallId: string): ScriptInvocation | undefined { + return this.previews.get(toolCallId); + } + set(toolCallId: string, preview: ScriptInvocation): void { this.previews.set(toolCallId, preview); } } describe("script preview events", () => { + it("holds formatted decoration until completion and expands to original source", async () => { + const bash = createNativeBashFeature(); + const theme = new Theme(TEST_THEME_COLORS, TEST_THEME_BACKGROUND_COLORS, "truecolor"); + const code = + 'import fs from "node:fs";\n\nconst entry = {\n alpha: 1,\n beta: 2,\n};\n\nconsole.log(entry);'; + const command = `node -e '${code}'`; + const executable = join( + dirname(createRequire(import.meta.url).resolve("oxfmt/package.json")), + "bin", + "oxfmt", + ); + const backend = createCommandScriptFormatter( + parseScriptFormatterCommandsValue({ + javascript: { kind: "javascript", command: [executable], imports: "omit" }, + }), + ); + if (backend === undefined) throw new Error("Expected a configured formatter"); + + let formatterCompleted = false; + const formatter: ScriptBlockFormatter = Object.assign( + async (input: ScriptBlockFormatterInput, options?: ScriptBlockFormatterOptions) => { + const output = await backend(input, options); + if (output !== undefined && options?.signal?.aborted !== true) + formatterCompleted = true; + + return output; + }, + { profileFor: (language: string) => backend.profileFor?.(language) }, + ); + let context: BuiltInRenderContext = { + args: { command }, + toolCallId: "display-call", + cwd: "/work", + lastComponent: undefined, + state: {}, + executionStarted: true, + argsComplete: true, + isPartial: true, + expanded: false, + showImages: false, + isError: false, + invalidate: () => {}, + }; + const render = (expanded = false) => + stripVTControlCharacters( + bash + .renderBashCall( + { command }, + theme, + { ...context, expanded }, + () => "block", + () => 30, + () => false, + () => "preserve", + () => "trailing", + ) + .render(120) + .join("\n"), + ); + try { + bash.schedule({ + toolCallId: context.toolCallId, + command, + formatter, + signal: undefined, + isCurrent: () => true, + invalidate: context.invalidate, + }); + + render(); + await vi.waitUntil( + () => formatterCompleted && getScriptFormatterWorkload().active === 0, + ); + expect(render()).toContain('│ import fs from "node:fs";'); + expect(render()).toContain("│ const entry = {\n"); + const result = { content: [{ type: "text" as const, text: "SAFE_RESULT" }] }; + context = { ...context, isPartial: false }; + bash.renderBashResult( + result, + { expanded: false, isPartial: false }, + theme, + context, + () => "block", + ); + await vi.waitUntil(() => render().includes("const entry = { alpha: 1, beta: 2 };")); + expect(render()).not.toContain("import fs"); + expect(render(true)).toContain('│ import fs from "node:fs";'); + expect(render(true)).toContain("│ const entry = {\n"); + expect(context.args).toEqual({ command }); + } finally { + bash.clear(); + await vi.waitUntil(() => getScriptFormatterWorkload().active === 0); + } + }); + + it("formats for the rendered content width and restores a single line after widening", async () => { + const bash = createNativeBashFeature(); + const theme = new Theme(TEST_THEME_COLORS, TEST_THEME_BACKGROUND_COLORS, "truecolor"); + const code = + 'console.log(["a moderately long sample value", "another moderately long sample value"]);'; + const command = `node -e '${code}'`; + const formatter: ScriptBlockFormatter = async (input, options) => { + const formatOptions: FormatConfig = {}; + if (options?.targetWidth !== undefined) { + formatOptions.printWidth = options.targetWidth; + } + + const formatted = await format("script.js", input.code, formatOptions); + return formatted.code; + }; + + let finishInvalidation = () => {}; + + const context: BuiltInRenderContext = { + args: { command }, + toolCallId: "width-call", + cwd: "/work", + lastComponent: undefined, + state: {}, + executionStarted: true, + argsComplete: true, + isPartial: false, + expanded: true, + showImages: false, + isError: false, + invalidate: () => finishInvalidation(), + }; + const render = (width: number) => + stripVTControlCharacters( + bash + .renderBashCall( + { command }, + theme, + context, + () => "auto", + () => 8, + () => false, + () => "auto", + () => "trailing", + ) + .render(width) + .join("\n"), + ); + bash.schedule({ + toolCallId: "width-call", + command, + formatter, + signal: undefined, + isCurrent: () => true, + invalidate: context.invalidate, + }); + const narrowed = new Promise((resolve) => { + finishInvalidation = resolve; + }); + render(44); + await narrowed; + expect(render(44)).toContain("│ console.log([\n"); + expect(render(44)).toContain('│ "a moderately long sample value",'); + const widened = new Promise((resolve) => { + finishInvalidation = resolve; + }); + render(100); + await widened; + expect(render(100)).toContain(code); + expect(render(100)).not.toContain("│ console.log([\n"); + bash.clear(); + }); + it("runs one formatter per finalized command", async () => { const bash = createNativeBashFeature(); const pending: Array<(code: string) => void> = []; diff --git a/extensions/pi-glowup/test/tools/third-party-renderers.test.ts b/extensions/pi-glowup/test/tools/third-party-renderers.test.ts index 512ab365..c9c2d173 100644 --- a/extensions/pi-glowup/test/tools/third-party-renderers.test.ts +++ b/extensions/pi-glowup/test/tools/third-party-renderers.test.ts @@ -1,4 +1,15 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { stripVTControlCharacters } from "node:util"; +import { createCodemodePreview } from "../../src/tools/external/codemode.ts"; +import { + createCommandScriptFormatter, + parseScriptFormatterCommandsValue, + type ScriptBlockFormatter, + type ScriptBlockFormatterInput, + type ScriptBlockFormatterOptions, +} from "../../src/tools/built-in/bash/formatter.ts"; import { visibleWidth } from "@earendil-works/pi-tui"; import { Type, type Static } from "typebox"; import { Value } from "typebox/value"; @@ -1220,6 +1231,115 @@ describe("third-party tool renderers", () => { expect(rendered).not.toContain("questions: 1 items"); }); + it("publishes Codemode decoration only after completion, uses content width, and expands original source", async () => { + const previews = createCodemodePreview(); + const executable = join( + dirname(createRequire(import.meta.url).resolve("oxfmt/package.json")), + "bin", + "oxfmt", + ); + const backend = createCommandScriptFormatter( + parseScriptFormatterCommandsValue({ + javascript: { kind: "javascript", command: [executable], imports: "omit" }, + }), + ); + if (backend === undefined) throw new Error("Expected formatter"); + + const restored = createCodemodePreview(() => backend); + let prepared = false; + const formatter: ScriptBlockFormatter = Object.assign( + async (input: ScriptBlockFormatterInput, options?: ScriptBlockFormatterOptions) => { + const output = await backend(input, options); + if (output !== undefined && options?.signal?.aborted !== true) prepared = true; + return output; + }, + { profileFor: (language: string) => backend.profileFor?.(language) }, + ); + const code = + 'import fs from "node:fs";\n\nconst entry = {\n alpha: 100,\n beta: 200,\n gamma: 300,\n};\n\ntext(entry);'; + const args = { code }; + const renderer = createThirdPartyToolRenderer("codemode", { codemodePreview: previews }); + let finished = false; + + const context = { + ...renderContext, + toolCallId: "profile-code", + args, + invalidate: () => {}, + }; + const render = (width = 120, expanded = false) => + stripVTControlCharacters( + renderer + .renderCall(args, plainTheme, { + ...context, + isPartial: !finished, + expanded, + ...(finished && { result: { content: [] } }), + }) + .render(width) + .join("\n"), + ); + try { + previews.schedule({ + toolCallId: context.toolCallId, + code, + formatter, + isCurrent: () => true, + invalidate: () => {}, + }); + + render(); + await vi.waitUntil(() => prepared); + expect(render()).toContain('import fs from "node:fs";'); + expect(render()).toContain("const entry = {\n"); + finished = true; + renderer.renderResult( + { content: [] }, + { expanded: false, isPartial: false }, + plainTheme, + context, + ); + await vi.waitUntil(() => + render().includes("const entry = { alpha: 100, beta: 200, gamma: 300 };"), + ); + expect(render()).not.toContain("import fs"); + expect(render(120, true)).toContain('import fs from "node:fs";'); + expect(render(120, true)).toContain("const entry = {\n"); + render(42); + await vi.waitUntil(() => render(42).includes("const entry = {\n")); + render(); + await vi.waitUntil(() => + render().includes("const entry = { alpha: 100, beta: 200, gamma: 300 };"), + ); + expect(args).toEqual({ code }); + const restoredRenderer = createThirdPartyToolRenderer("codemode", { + codemodePreview: restored, + }); + const renderRestored = (expanded = false) => + stripVTControlCharacters( + restoredRenderer + .renderCall(args, plainTheme, { + ...context, + toolCallId: "restored-profile", + expanded, + result: { content: [] }, + }) + .render(120) + .join("\n"), + ); + expect(renderRestored()).toContain('import fs from "node:fs";'); + await vi.waitUntil(() => + renderRestored().includes("const entry = { alpha: 100, beta: 200, gamma: 300 };"), + ); + expect(renderRestored()).not.toContain("import fs"); + expect(renderRestored(true)).toContain('import fs from "node:fs";'); + expect(renderRestored(true)).toContain("const entry = {\n"); + } finally { + restored.clear(); + previews.clear(); + } + }); + it("shows codemode source and nested outcomes without script envelopes or raw result JSON", () => { const renderer = createThirdPartyToolRenderer("codemode"); const script = "const result = await tools.read({path: '/tmp/example'});\ntext(result);";