diff --git a/packages/cso-cli/src/bindings.ts b/packages/cso-cli/src/bindings.ts index 0949fd2..49be630 100644 --- a/packages/cso-cli/src/bindings.ts +++ b/packages/cso-cli/src/bindings.ts @@ -1,4 +1,4 @@ -import { spawnSync } from 'node:child_process'; +import { runPythonProcess } from './python-process.ts'; export const bindingsHelp = `Usage: cso bindings [--check] @@ -10,15 +10,11 @@ Exit codes: 0 current/generated, 1 stale/source/write failure, 2 invalid usage. `; export function bindingsCommand(args: string[]): number { - const environment = { ...process.env }; - for (const key of ['PYTHONPATH', 'PYTHONHOME', 'NODE_PATH']) { - Reflect.deleteProperty(environment, key); - } - const result = spawnSync( - process.env.PYTHON ?? 'python3', - ['-I', '-m', 'cso_python', 'bindings', ...args], - { env: environment, maxBuffer: 64 * 1024 * 1024 }, - ); + const result = runPythonProcess({ + command: 'bindings', + args, + maxBuffer: 64 * 1024 * 1024, + }); if (result.stderr) process.stderr.write(result.stderr); if (result.error || result.signal) { process.stdout.write( diff --git a/packages/cso-cli/src/dev-runtime.ts b/packages/cso-cli/src/dev-runtime.ts index b08580e..b1c535e 100644 --- a/packages/cso-cli/src/dev-runtime.ts +++ b/packages/cso-cli/src/dev-runtime.ts @@ -1,4 +1,3 @@ -import { spawnSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { CalculationDefinitionResponseSchema, @@ -16,6 +15,7 @@ import { jsonBytes, type PresentationMapping } from './evidence.ts'; import { renderPreparedPdf, LayoutInspectionError } from './pdf-rendering.ts'; import { buildPreparedHtml } from './prepared-html.ts'; import { parseStrictJson } from './strict-json.ts'; +import { runPythonProcess } from './python-process.ts'; import { executeAndVerify, type VerifiedCapture } from './verification.ts'; import type { VerifiedOptions } from './verified-arguments.ts'; @@ -54,27 +54,13 @@ const defaults: DevLimits = { maximumAgeMs: 30 * 60_000, }; function describe(options: DevTarget): CalculationDefinition { - const env = { ...process.env }; - for (const key of ['PYTHONPATH', 'PYTHONHOME', 'NODE_PATH']) - Reflect.deleteProperty(env, key); - const result = spawnSync( - process.env.PYTHON ?? 'python3', - [ - '-I', - '-m', - 'cso_python', - 'describe', - options.sourcePath, - '--function', - options.functionName, - ], - { - env, - timeout: 30_000, - killSignal: 'SIGKILL', - maxBuffer: 4 * 1024 * 1024, - }, - ); + const result = runPythonProcess({ + command: 'describe', + args: [options.sourcePath, '--function', options.functionName], + timeout: 30_000, + killSignal: 'SIGKILL', + maxBuffer: 4 * 1024 * 1024, + }); if (result.error || result.signal) throw new HttpError( 422, diff --git a/packages/cso-cli/src/development.ts b/packages/cso-cli/src/development.ts index 7ab4876..560cdc5 100644 --- a/packages/cso-cli/src/development.ts +++ b/packages/cso-cli/src/development.ts @@ -1,5 +1,4 @@ import { stringifyJson } from './json.ts'; -import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, @@ -22,6 +21,8 @@ import { type SheetDocument, } from '@cs-object/core'; import { UsageError, type DevelopmentOptions } from './arguments.ts'; +import { runPythonProcess } from './python-process.ts'; +export { isolatedEnvironment } from './python-process.ts'; export { parseCliArgs } from './arguments.ts'; export type AnnotatedPythonPdfOptions = DevelopmentOptions; export interface PythonExporterOptions { @@ -42,13 +43,6 @@ const escapeHtml = (value: string): string => .replace(//g, '>') .replace(/"/g, '"'); -export function isolatedEnvironment(): NodeJS.ProcessEnv { - const environment = { ...process.env }; - Reflect.deleteProperty(environment, 'NODE_PATH'); - Reflect.deleteProperty(environment, 'PYTHONPATH'); - Reflect.deleteProperty(environment, 'PYTHONHOME'); - return environment; -} export function writeAtomically( outPath: string, bytes: string | Uint8Array, @@ -70,32 +64,25 @@ export const runPythonExporter = ({ inputs, sourcePath, }: PythonExporterOptions): CalculationSourceObject => { - const result = spawnSync( - process.env.PYTHON ?? 'python3', - [ - '-I', - '-m', - 'cso_python', - 'export', + const result = runPythonProcess({ + command: 'export', + args: [ sourcePath, '--function', functionName, '--inputs-json', stringifyJson(inputs), ], - { - encoding: 'utf8', - env: isolatedEnvironment(), - maxBuffer: 32 * 1024 * 1024, - }, - ); + maxBuffer: 32 * 1024 * 1024, + }); if (result.error) { throw result.error; } if (result.status !== 0) { - const message = result.stderr.trim() || 'Python exporter failed'; + const message = + result.stderr.toString('utf8').trim() || 'Python exporter failed'; if (result.status === 2) throw new UsageError(message); throw new Error(message); } @@ -104,7 +91,9 @@ export const runPythonExporter = ({ process.stderr.write(result.stderr); } - return CalculationSourceObjectSchema.parse(JSON.parse(result.stdout)); + return CalculationSourceObjectSchema.parse( + JSON.parse(result.stdout.toString('utf8')), + ); }; export const renderFormulaSheetHtml = ( diff --git a/packages/cso-cli/src/execution.ts b/packages/cso-cli/src/execution.ts index 0cff35e..2048d6b 100644 --- a/packages/cso-cli/src/execution.ts +++ b/packages/cso-cli/src/execution.ts @@ -1,5 +1,4 @@ import { stringifyJson } from './json.ts'; -import { spawnSync } from 'node:child_process'; import { ExecutionResponseSchema, contractIssuesToDiagnostics, @@ -7,6 +6,7 @@ import { type Diagnostic, } from '@cs-object/core'; import { parseStrictJson } from './strict-json.ts'; +import { runPythonProcess } from './python-process.ts'; import type { VerifiedOptions } from './verified-arguments.ts'; type ExecutionAttempt = @@ -24,30 +24,19 @@ export function runPythonExecution( options: VerifiedOptions, timeoutMs?: number, ): ExecutionAttempt { - const environment = { ...process.env }; - for (const key of ['PYTHONPATH', 'PYTHONHOME', 'NODE_PATH']) { - Reflect.deleteProperty(environment, key); - } - const result = spawnSync( - process.env.PYTHON ?? 'python3', - [ - '-I', - '-m', - 'cso_python', - 'execute', + const result = runPythonProcess({ + command: 'execute', + args: [ options.sourcePath, '--function', options.functionName, '--inputs-json', stringifyJson(options.inputs), ], - { - env: environment, - maxBuffer: 64 * 1024 * 1024, - timeout: timeoutMs, - killSignal: 'SIGKILL', - }, - ); + maxBuffer: 64 * 1024 * 1024, + timeout: timeoutMs, + killSignal: 'SIGKILL', + }); if (result.stderr) { process.stderr.write(result.stderr); } diff --git a/packages/cso-cli/src/python-process.ts b/packages/cso-cli/src/python-process.ts new file mode 100644 index 0000000..cba16ac --- /dev/null +++ b/packages/cso-cli/src/python-process.ts @@ -0,0 +1,28 @@ +import { spawnSync } from 'node:child_process'; + +type PythonProcessRequest = { + command: 'execute' | 'describe' | 'bindings' | 'export'; + args: string[]; + maxBuffer: number; + timeout?: number; + killSignal?: NodeJS.Signals; +}; + +export function isolatedEnvironment(): NodeJS.ProcessEnv { + const environment = { ...process.env }; + for (const key of ['PYTHONPATH', 'PYTHONHOME', 'NODE_PATH']) + Reflect.deleteProperty(environment, key); + return environment; +} + +export function runPythonProcess({ + command, + args, + ...limits +}: PythonProcessRequest) { + return spawnSync( + process.env.PYTHON ?? 'python3', + ['-I', '-X', 'utf8', '-m', 'cso_python', command, ...args], + { env: isolatedEnvironment(), ...limits }, + ); +} diff --git a/tests/integration/cli-python-utf8.test.ts b/tests/integration/cli-python-utf8.test.ts new file mode 100644 index 0000000..a892b76 --- /dev/null +++ b/tests/integration/cli-python-utf8.test.ts @@ -0,0 +1,134 @@ +import { spawnSync } from 'node:child_process'; +import { + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; +import { z } from 'zod'; +import { createDevRuntime } from '../../packages/cso-cli/src/dev-runtime.ts'; +import { runPythonExporter } from '../../packages/cso-cli/src/development.ts'; +import { executeAndVerify } from '../../packages/cso-cli/src/verification.ts'; + +const directories: string[] = []; +const cli = fileURLToPath( + new URL('../../packages/cso-cli/dist/cli.js', import.meta.url), +); + +beforeEach(() => { + const directory = mkdtempSync(join(tmpdir(), 'cso-python-environment-')); + directories.push(directory); + writeFileSync( + join(directory, 'cso_python.py'), + 'raise RuntimeError("Untrusted Python path")\n', + ); + vi.stubEnv('PYTHONPATH', directory); + vi.stubEnv('PYTHONHOME', join(directory, 'missing-python-home')); + vi.stubEnv('NODE_PATH', directory); + vi.stubEnv('PYTHONIOENCODING', 'ascii'); + vi.stubEnv('PYTHONUTF8', '0'); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + for (const directory of directories.splice(0)) + rmSync(directory, { recursive: true, force: true }); +}); + +function fixture() { + const directory = realpathSync(mkdtempSync(join(tmpdir(), 'cso café 梁 '))); + directories.push(directory); + const sourcePath = join(directory, 'quantity.cso.py'); + writeFileSync( + sourcePath, + `from typing import Annotated +from cso_python import calculation, section, symbol +@calculation(id="unicode", title="Épaisseur 梁", metadata={"author": "Zoë 李"}) +@section(id="main", title="Mesure 梁", root=True) +def calculate(width: Annotated[float, symbol(glyph="w_{test}", description="Largeur café 梁", unit="mm")] = 2): + doubled: Annotated[float, symbol(glyph="d_{test}", description="Épaisseur doublée 梁", unit="mm")] = width * 2 + return {"doublée 梁": doubled} +`, + ); + return { directory, sourcePath, functionName: 'calculate' }; +} + +test('verified execution retains Unicode author fields and original UTF-8 bytes', () => { + const result = executeAndVerify({ + ...fixture(), + command: 'verify', + inputs: {}, + }); + expect(result.kind).toBe('verified'); + if (result.kind !== 'verified') + throw new Error('Expected verified execution'); + expect(result.capture.execution.cso.title).toBe('Épaisseur 梁'); + expect(result.capture.execution.cso.source.metadata?.author).toBe('Zoë 李'); + expect(result.capture.execution.authoring?.outputs).toMatchObject([ + { name: 'doublée 梁', value: 4 }, + ]); + expect(result.capture.executionBytes.includes(Buffer.from('Zoë 李'))).toBe( + true, + ); +}); + +test('the dev definition preserves Unicode descriptions and public output names', async () => { + const runtime = createDevRuntime(fixture()); + try { + const definition = runtime.definition(); + expect(definition.inputs).toMatchObject([ + { name: 'width', description: 'Largeur café 梁', default: 2 }, + ]); + expect(definition.outputs).toMatchObject([ + { name: 'doublée 梁', description: 'Épaisseur doublée 梁' }, + ]); + expect(runtime.calculate({ definition, inputs: { width: 3 } })).toEqual({ + 'doublée 梁': 6, + }); + } finally { + await runtime.close(); + } +}); + +test('bindings forward UTF-8 JSON and generate handles in a Unicode directory', () => { + const { directory } = fixture(); + for (const extra of [[], ['--check']]) { + const result = spawnSync(process.execPath, [ + cli, + 'bindings', + directory, + ...extra, + ]); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr.toString('utf8')).toBe(0); + const stdout = new TextDecoder('utf-8', { fatal: true }).decode( + result.stdout, + ); + const response = z + .object({ ok: z.boolean(), directory: z.string() }) + .parse(JSON.parse(stdout)); + expect(response.ok).toBe(true); + expect(realpathSync.native(response.directory)).toBe( + realpathSync.native(directory), + ); + expect(stdout).toContain('café 梁'); + } + expect( + readFileSync(join(directory, '_cso_bindings', 'quantity.pyi'), 'utf8'), + ).toContain('doublée 梁'); +}); + +test('the legacy exporter preserves Unicode JSON and decoded failure messages', () => { + const options = { ...fixture(), inputs: {} }; + const document = runPythonExporter(options); + expect(document.title).toBe('Épaisseur 梁'); + expect(document.source.metadata?.author).toBe('Zoë 李'); + expect(() => + runPythonExporter({ ...options, functionName: 'absent_é梁' }), + ).toThrow('absent_é梁'); +}); diff --git a/tests/integration/installed/pdf-consumer.mjs b/tests/integration/installed/pdf-consumer.mjs index e1390bc..76d1e67 100644 --- a/tests/integration/installed/pdf-consumer.mjs +++ b/tests/integration/installed/pdf-consumer.mjs @@ -40,7 +40,7 @@ function run(name, args, expected = 0, extraEnv = {}, outPath = resolve(`pdf-out assert.equal(result.stdout.toString().trim().split('\n').length, 1); const calls = readFileSync('pdf-invocations.jsonl', 'utf8').trim().split('\n').map(JSON.parse); assert.equal(calls.length, 1, name); - assert.deepEqual(calls[0].slice(0, 4), ['-I', '-m', 'cso_python', 'execute']); + assert.deepEqual(calls[0].slice(0, 6), ['-I', '-X', 'utf8', '-m', 'cso_python', 'execute']); assert.equal(report.command, 'pdf'); assert.equal(report.ok, expected === 0); let manifest; diff --git a/tests/integration/installed/verify-consumer.mjs b/tests/integration/installed/verify-consumer.mjs index 2cf9c9c..e1b9ec6 100644 --- a/tests/integration/installed/verify-consumer.mjs +++ b/tests/integration/installed/verify-consumer.mjs @@ -82,7 +82,7 @@ function run(name, args, expected = 0, executions = 1, extraEnv = {}) { assert.equal(calls.length, executions); if (!executions) assert(!readdirSync('.').includes('invocations.jsonl')); for (const call of calls) - assert.deepEqual(call.slice(0, 4), ['-I', '-m', 'cso_python', 'execute']); + assert.deepEqual(call.slice(0, 6), ['-I', '-X', 'utf8', '-m', 'cso_python', 'execute']); if (executions && !extraEnv.PYTHON) assert.match(result.stderr, /observer operational stderr/); assert.equal(report.checks.documentContent.status, 'not_applicable');