fix(cli): run Python subprocesses in UTF-8 mode

packages/cso-cli/src/bindings.ts

@@ -1,4 +1,4 @@
-import { spawnSync } from 'node:child_process';
+import { runPythonProcess } from './python-process.ts';
 
 export const bindingsHelp = `Usage: cso bindings <directory> [--check]
 
@@ -10,15 +10,11 @@ Exit codes: 0 current/generated, 1 stale/source/write failure, 2 invalid usage.
 `;
 
 export function bindingsCommand(args: string[]): number {
-  const environment = { ...process.env };
-  for (const key of ['PYTHONPATH', 'PYTHONHOME', 'NODE_PATH']) {
-    Reflect.deleteProperty(environment, key);
-  }
-  const result = spawnSync(
-    process.env.PYTHON ?? 'python3',
-    ['-I', '-m', 'cso_python', 'bindings', ...args],
-    { env: environment, maxBuffer: 64 * 1024 * 1024 },
-  );
+  const result = runPythonProcess({
+    command: 'bindings',
+    args,
+    maxBuffer: 64 * 1024 * 1024,
+  });
   if (result.stderr) process.stderr.write(result.stderr);
   if (result.error || result.signal) {
     process.stdout.write(

packages/cso-cli/src/dev-runtime.ts

@@ -1,4 +1,3 @@
-import { spawnSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import {
   CalculationDefinitionResponseSchema,
@@ -16,6 +15,7 @@ import { jsonBytes, type PresentationMapping } from './evidence.ts';
 import { renderPreparedPdf, LayoutInspectionError } from './pdf-rendering.ts';
 import { buildPreparedHtml } from './prepared-html.ts';
 import { parseStrictJson } from './strict-json.ts';
+import { runPythonProcess } from './python-process.ts';
 import { executeAndVerify, type VerifiedCapture } from './verification.ts';
 import type { VerifiedOptions } from './verified-arguments.ts';
 
@@ -54,27 +54,13 @@ const defaults: DevLimits = {
   maximumAgeMs: 30 * 60_000,
 };
 function describe(options: DevTarget): CalculationDefinition {
-  const env = { ...process.env };
-  for (const key of ['PYTHONPATH', 'PYTHONHOME', 'NODE_PATH'])
-    Reflect.deleteProperty(env, key);
-  const result = spawnSync(
-    process.env.PYTHON ?? 'python3',
-    [
-      '-I',
-      '-m',
-      'cso_python',
-      'describe',
-      options.sourcePath,
-      '--function',
-      options.functionName,
-    ],
-    {
-      env,
-      timeout: 30_000,
-      killSignal: 'SIGKILL',
-      maxBuffer: 4 * 1024 * 1024,
-    },
-  );
+  const result = runPythonProcess({
+    command: 'describe',
+    args: [options.sourcePath, '--function', options.functionName],
+    timeout: 30_000,
+    killSignal: 'SIGKILL',
+    maxBuffer: 4 * 1024 * 1024,
+  });
   if (result.error || result.signal)
     throw new HttpError(
       422,

packages/cso-cli/src/development.ts

@@ -1,5 +1,4 @@
 import { stringifyJson } from './json.ts';
-import { spawnSync } from 'node:child_process';
 import {
   existsSync,
   mkdirSync,
@@ -22,6 +21,8 @@ import {
   type SheetDocument,
 } from '@cs-object/core';
 import { UsageError, type DevelopmentOptions } from './arguments.ts';
+import { runPythonProcess } from './python-process.ts';
+export { isolatedEnvironment } from './python-process.ts';
 export { parseCliArgs } from './arguments.ts';
 export type AnnotatedPythonPdfOptions = DevelopmentOptions;
 export interface PythonExporterOptions {
@@ -42,13 +43,6 @@ const escapeHtml = (value: string): string =>
     .replace(/</g, '&lt;')
     .replace(/>/g, '&gt;')
     .replace(/"/g, '&quot;');
-export function isolatedEnvironment(): NodeJS.ProcessEnv {
-  const environment = { ...process.env };
-  Reflect.deleteProperty(environment, 'NODE_PATH');
-  Reflect.deleteProperty(environment, 'PYTHONPATH');
-  Reflect.deleteProperty(environment, 'PYTHONHOME');
-  return environment;
-}
 export function writeAtomically(
   outPath: string,
   bytes: string | Uint8Array,
@@ -70,32 +64,25 @@ export const runPythonExporter = ({
   inputs,
   sourcePath,
 }: PythonExporterOptions): CalculationSourceObject => {
-  const result = spawnSync(
-    process.env.PYTHON ?? 'python3',
-    [
-      '-I',
-      '-m',
-      'cso_python',
-      'export',
+  const result = runPythonProcess({
+    command: 'export',
+    args: [
       sourcePath,
       '--function',
       functionName,
       '--inputs-json',
       stringifyJson(inputs),
     ],
-    {
-      encoding: 'utf8',
-      env: isolatedEnvironment(),
-      maxBuffer: 32 * 1024 * 1024,
-    },
-  );
+    maxBuffer: 32 * 1024 * 1024,
+  });
 
   if (result.error) {
     throw result.error;
   }
 
   if (result.status !== 0) {
-    const message = result.stderr.trim() || 'Python exporter failed';
+    const message =
+      result.stderr.toString('utf8').trim() || 'Python exporter failed';
     if (result.status === 2) throw new UsageError(message);
     throw new Error(message);
   }
@@ -104,7 +91,9 @@ export const runPythonExporter = ({
     process.stderr.write(result.stderr);
   }
 
-  return CalculationSourceObjectSchema.parse(JSON.parse(result.stdout));
+  return CalculationSourceObjectSchema.parse(
+    JSON.parse(result.stdout.toString('utf8')),
+  );
 };
 
 export const renderFormulaSheetHtml = (

packages/cso-cli/src/execution.ts

@@ -1,12 +1,12 @@
 import { stringifyJson } from './json.ts';
-import { spawnSync } from 'node:child_process';
 import {
   ExecutionResponseSchema,
   contractIssuesToDiagnostics,
   type ExecutionResponse,
   type Diagnostic,
 } from '@cs-object/core';
 import { parseStrictJson } from './strict-json.ts';
+import { runPythonProcess } from './python-process.ts';
 import type { VerifiedOptions } from './verified-arguments.ts';
 
 type ExecutionAttempt =
@@ -24,30 +24,19 @@ export function runPythonExecution(
   options: VerifiedOptions,
   timeoutMs?: number,
 ): ExecutionAttempt {
-  const environment = { ...process.env };
-  for (const key of ['PYTHONPATH', 'PYTHONHOME', 'NODE_PATH']) {
-    Reflect.deleteProperty(environment, key);
-  }
-  const result = spawnSync(
-    process.env.PYTHON ?? 'python3',
-    [
-      '-I',
-      '-m',
-      'cso_python',
-      'execute',
+  const result = runPythonProcess({
+    command: 'execute',
+    args: [
       options.sourcePath,
       '--function',
       options.functionName,
       '--inputs-json',
       stringifyJson(options.inputs),
     ],
-    {
-      env: environment,
-      maxBuffer: 64 * 1024 * 1024,
-      timeout: timeoutMs,
-      killSignal: 'SIGKILL',
-    },
-  );
+    maxBuffer: 64 * 1024 * 1024,
+    timeout: timeoutMs,
+    killSignal: 'SIGKILL',
+  });
   if (result.stderr) {
     process.stderr.write(result.stderr);
   }

packages/cso-cli/src/python-process.ts

@@ -0,0 +1,28 @@
+import { spawnSync } from 'node:child_process';
+
+type PythonProcessRequest = {
+  command: 'execute' | 'describe' | 'bindings' | 'export';
+  args: string[];
+  maxBuffer: number;
+  timeout?: number;
+  killSignal?: NodeJS.Signals;
+};
+
+export function isolatedEnvironment(): NodeJS.ProcessEnv {
+  const environment = { ...process.env };
+  for (const key of ['PYTHONPATH', 'PYTHONHOME', 'NODE_PATH'])
+    Reflect.deleteProperty(environment, key);
+  return environment;
+}
+
+export function runPythonProcess({
+  command,
+  args,
+  ...limits
+}: PythonProcessRequest) {
+  return spawnSync(
+    process.env.PYTHON ?? 'python3',
+    ['-I', '-X', 'utf8', '-m', 'cso_python', command, ...args],
+    { env: isolatedEnvironment(), ...limits },
+  );
+}

tests/integration/cli-python-utf8.test.ts

@@ -0,0 +1,134 @@
+import { spawnSync } from 'node:child_process';
+import {
+  mkdtempSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
+import { z } from 'zod';
+import { createDevRuntime } from '../../packages/cso-cli/src/dev-runtime.ts';
+import { runPythonExporter } from '../../packages/cso-cli/src/development.ts';
+import { executeAndVerify } from '../../packages/cso-cli/src/verification.ts';
+
+const directories: string[] = [];
+const cli = fileURLToPath(
+  new URL('../../packages/cso-cli/dist/cli.js', import.meta.url),
+);
+
+beforeEach(() => {
+  const directory = mkdtempSync(join(tmpdir(), 'cso-python-environment-'));
+  directories.push(directory);
+  writeFileSync(
+    join(directory, 'cso_python.py'),
+    'raise RuntimeError("Untrusted Python path")\n',
+  );
+  vi.stubEnv('PYTHONPATH', directory);
+  vi.stubEnv('PYTHONHOME', join(directory, 'missing-python-home'));
+  vi.stubEnv('NODE_PATH', directory);
+  vi.stubEnv('PYTHONIOENCODING', 'ascii');
+  vi.stubEnv('PYTHONUTF8', '0');
+});
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+  for (const directory of directories.splice(0))
+    rmSync(directory, { recursive: true, force: true });
+});
+
+function fixture() {
+  const directory = realpathSync(mkdtempSync(join(tmpdir(), 'cso café 梁 ')));
+  directories.push(directory);
+  const sourcePath = join(directory, 'quantity.cso.py');
+  writeFileSync(
+    sourcePath,
+    `from typing import Annotated
+from cso_python import calculation, section, symbol
+@calculation(id="unicode", title="Épaisseur 梁", metadata={"author": "Zoë 李"})
+@section(id="main", title="Mesure 梁", root=True)
+def calculate(width: Annotated[float, symbol(glyph="w_{test}", description="Largeur café 梁", unit="mm")] = 2):
+    doubled: Annotated[float, symbol(glyph="d_{test}", description="Épaisseur doublée 梁", unit="mm")] = width * 2
+    return {"doublée 梁": doubled}
+`,
+  );
+  return { directory, sourcePath, functionName: 'calculate' };
+}
+
+test('verified execution retains Unicode author fields and original UTF-8 bytes', () => {
+  const result = executeAndVerify({
+    ...fixture(),
+    command: 'verify',
+    inputs: {},
+  });
+  expect(result.kind).toBe('verified');
+  if (result.kind !== 'verified')
+    throw new Error('Expected verified execution');
+  expect(result.capture.execution.cso.title).toBe('Épaisseur 梁');
+  expect(result.capture.execution.cso.source.metadata?.author).toBe('Zoë 李');
+  expect(result.capture.execution.authoring?.outputs).toMatchObject([
+    { name: 'doublée 梁', value: 4 },
+  ]);
+  expect(result.capture.executionBytes.includes(Buffer.from('Zoë 李'))).toBe(
+    true,
+  );
+});
+
+test('the dev definition preserves Unicode descriptions and public output names', async () => {
+  const runtime = createDevRuntime(fixture());
+  try {
+    const definition = runtime.definition();
+    expect(definition.inputs).toMatchObject([
+      { name: 'width', description: 'Largeur café 梁', default: 2 },
+    ]);
+    expect(definition.outputs).toMatchObject([
+      { name: 'doublée 梁', description: 'Épaisseur doublée 梁' },
+    ]);
+    expect(runtime.calculate({ definition, inputs: { width: 3 } })).toEqual({
+      'doublée 梁': 6,
+    });
+  } finally {
+    await runtime.close();
+  }
+});
+
+test('bindings forward UTF-8 JSON and generate handles in a Unicode directory', () => {
+  const { directory } = fixture();
+  for (const extra of [[], ['--check']]) {
+    const result = spawnSync(process.execPath, [
+      cli,
+      'bindings',
+      directory,
+      ...extra,
+    ]);
+    expect(result.error).toBeUndefined();
+    expect(result.status, result.stderr.toString('utf8')).toBe(0);
+    const stdout = new TextDecoder('utf-8', { fatal: true }).decode(
+      result.stdout,
+    );
+    const response = z
+      .object({ ok: z.boolean(), directory: z.string() })
+      .parse(JSON.parse(stdout));
+    expect(response.ok).toBe(true);
+    expect(realpathSync.native(response.directory)).toBe(
+      realpathSync.native(directory),
+    );
+    expect(stdout).toContain('café 梁');
+  }
+  expect(
+    readFileSync(join(directory, '_cso_bindings', 'quantity.pyi'), 'utf8'),
+  ).toContain('doublée 梁');
+});
+
+test('the legacy exporter preserves Unicode JSON and decoded failure messages', () => {
+  const options = { ...fixture(), inputs: {} };
+  const document = runPythonExporter(options);
+  expect(document.title).toBe('Épaisseur 梁');
+  expect(document.source.metadata?.author).toBe('Zoë 李');
+  expect(() =>
+    runPythonExporter({ ...options, functionName: 'absent_é梁' }),
+  ).toThrow('absent_é梁');
+});

tests/integration/installed/pdf-consumer.mjs

@@ -40,7 +40,7 @@ function run(name, args, expected = 0, extraEnv = {}, outPath = resolve(`pdf-out
   assert.equal(result.stdout.toString().trim().split('\n').length, 1);
   const calls = readFileSync('pdf-invocations.jsonl', 'utf8').trim().split('\n').map(JSON.parse);
   assert.equal(calls.length, 1, name);
-  assert.deepEqual(calls[0].slice(0, 4), ['-I', '-m', 'cso_python', 'execute']);
+  assert.deepEqual(calls[0].slice(0, 6), ['-I', '-X', 'utf8', '-m', 'cso_python', 'execute']);
   assert.equal(report.command, 'pdf');
   assert.equal(report.ok, expected === 0);
   let manifest;

tests/integration/installed/verify-consumer.mjs

@@ -82,7 +82,7 @@ function run(name, args, expected = 0, executions = 1, extraEnv = {}) {
   assert.equal(calls.length, executions);
   if (!executions) assert(!readdirSync('.').includes('invocations.jsonl'));
   for (const call of calls)
-    assert.deepEqual(call.slice(0, 4), ['-I', '-m', 'cso_python', 'execute']);
+    assert.deepEqual(call.slice(0, 6), ['-I', '-X', 'utf8', '-m', 'cso_python', 'execute']);
   if (executions && !extraEnv.PYTHON)
     assert.match(result.stderr, /observer operational stderr/);
   assert.equal(report.checks.documentContent.status, 'not_applicable');