Add a reusable Graphile performance harness

.github/workflows/run-tests.yaml

@@ -109,7 +109,7 @@ jobs:
           - batch: packages-core
             packages: 'packages/url-domains packages/coerce packages/csrf packages/oauth packages/12factor-env packages/orm packages/express-context packages/errors packages/llm-env packages/node-type-registry packages/query-spec packages/server-utils packages/site-deploy examples/site-deploy-ssg postgres/pg-cache postgres/pg-env'
           - batch: packages-services
-            packages: 'packages/postmaster packages/smtppostmaster packages/csv-to-pg packages/cli postgres/pgsql-client postgres/pg-ast'
+            packages: 'packages/postmaster packages/smtppostmaster packages/csv-to-pg packages/cli packages/perf-harness postgres/pgsql-client postgres/pg-ast'
           - batch: graphql
             packages: 'graphql/query graphql/codegen'
           - batch: graphile-unit

packages/perf-harness/README.md

@@ -0,0 +1,82 @@
+# Graphile performance harness
+
+Reusable infrastructure for measuring Graphile schema builds in fresh Node
+processes. The core accepts any list of serializable benchmark cases; it does not
+interpret case names or optimization-specific configuration.
+
+Each measurement receives a new PID, starts Node with `--expose-gc`, runs a
+deterministic GC sequence, records build time and memory metrics, validates a
+runtime query, and reports a schema hash. Cases can opt into schema equivalence
+groups and provide their own lifecycle validation through the worker result.
+
+## Running locally
+
+Build the package, then invoke its local CLI script from the workspace root:
+
+```sh
+pnpm --filter @constructive-io/perf-harness build
+pnpm --filter @constructive-io/perf-harness cperf prepare \
+  --database-url 'postgresql:///benchmark' \
+  --schema cperf_example --tables 4
+```
+
+Use an existing local benchmark database. The equivalent direct CLI entry is
+`node packages/perf-harness/dist/cli.js`; pass `run` with the suite's `--cases`
+and `--worker` arguments to execute measurements.
+
+`makage` assembles the package in `dist`, where `index.js` is the CommonJS library
+entry, `esm/index.js` is the ESM library entry for bundlers, and `cli.js` is the
+`cperf` executable. The package manifest's entry paths are relative to that
+artifact directory. The private workspace package uses the local `cperf` script
+above. Importing either library entry does not start the CLI.
+
+`pnpm --filter @constructive-io/perf-harness test` rebuilds the package before
+running unit tests and smoke tests against the generated library and CLI entries.
+
+## Extending the harness
+
+Define a suite and provide a dedicated worker entry point:
+
+```ts
+const suite = {
+  name: 'example',
+  cases: [
+    {
+      name: 'baseline',
+      workerConfig: { schemas: ['cperf_example'] },
+      expectedSchemaGroup: 'example-schema',
+    },
+  ],
+};
+
+await runBenchmarkSuite(suite, options, workerPath);
+```
+
+`workerConfig` must be JSON-serializable. Logic is implemented in the worker
+entry rather than serializing functions across process boundaries.
+
+Each worker has a five-minute wall-clock deadline, including startup, database
+access, validation, and cleanup. Set `options.workerTimeoutMs` or pass
+`--worker-timeout-ms` to `cperf run` to override it with a positive integer
+(maximum 2,147,483,647 ms). The effective deadline is recorded in `report.config`;
+it does not change the build-only `buildMs` measurement.
+
+A timed-out worker is killed and must finish closing before the next case starts.
+Its failure is recorded even if it already printed a successful result. If its
+process and stdio cannot be confirmed closed within another five seconds, the
+suite stops scheduling cases and writes a failed report containing the completed
+runs. Earlier successful samples may remain in summaries, but the report's
+validation fails. The CLI exits nonzero for either kind of failure.
+
+The package includes `stock-worker.js` as a minimal upstream Graphile baseline.
+It emits one result after releasing its Graphile service. A measurement or release
+failure fails the run; if both fail, the error report retains both diagnostics in
+that order.
+The top-level commands require `--database-url`; the runner forwards it and the
+opaque case configuration to each short-lived worker as CLI arguments. Database
+credentials are redacted from worker failures and JSON reports. This harness is
+intended for local development on a trusted machine because command arguments
+may be visible to other local processes.
+
+The PostgreSQL fixture command only creates a previously absent schema whose
+name starts with `cperf_`; it never drops or replaces schemas.

packages/perf-harness/__tests__/entrypoints.test.ts

@@ -0,0 +1,116 @@
+import { spawnSync } from 'node:child_process';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import type { BenchmarkReport } from '../src/types';
+
+describe('built package entry points', () => {
+  const packageRoot = resolve(__dirname, '..');
+  const artifactRoot = resolve(packageRoot, 'dist');
+  let manifest: { main: string; module: string; bin: { cperf: string } };
+
+  beforeAll(async () => {
+    manifest = JSON.parse(
+      await readFile(resolve(artifactRoot, 'package.json'), 'utf8')
+    );
+  });
+
+  const node = (args: string[], input?: string) =>
+    spawnSync(process.execPath, args, {
+      cwd: packageRoot,
+      encoding: 'utf8',
+      input,
+      timeout: 15_000,
+    });
+
+  test('imports the CommonJS library without running the CLI', () => {
+    const entry = resolve(artifactRoot, manifest.main);
+    const result = node([
+      '--eval',
+      `const assert = require('node:assert/strict');
+       const library = require(${JSON.stringify(entry)});
+       assert.equal(typeof library.runBenchmarkSuite, 'function');
+       assert.equal(typeof library.prepareFixture, 'function');`,
+    ]);
+    expect(result.error).toBeUndefined();
+    expect(result.status).toBe(0);
+    expect(result.stdout).toBe('');
+    expect(result.stderr).toBe('');
+  });
+
+  test('imports the emitted ESM library without CommonJS globals or CLI startup', () => {
+    const entry = pathToFileURL(resolve(artifactRoot, manifest.module)).href;
+    const result = node(
+      [
+        '--no-warnings',
+        '--experimental-loader',
+        pathToFileURL(resolve(__dirname, 'fixtures/esm-loader.mjs')).href,
+        '--input-type=module',
+      ],
+      `import assert from 'node:assert/strict';
+       import * as library from ${JSON.stringify(entry)};
+       assert.equal(typeof require, 'undefined');
+       assert.equal(typeof module, 'undefined');
+       assert.equal(typeof library.runBenchmarkSuite, 'function');
+       assert.equal(typeof library.prepareFixture, 'function');`
+    );
+    expect(result.error).toBeUndefined();
+    expect(result.stderr).toBe('');
+    expect(result.status).toBe(0);
+    expect(result.stdout).toBe('');
+  });
+
+  test('runs a benchmark through the built manifest bin and writes its report', async () => {
+    const directory = await mkdtemp(resolve(tmpdir(), 'cperf-entry-'));
+    const output = resolve(directory, 'report.json');
+    const databaseUrl = 'postgres://[email protected]/database';
+    const cases = [
+      { name: 'baseline', workerConfig: { value: 1, schemaHash: 'same' } },
+    ];
+    try {
+      const result = node([
+        resolve(artifactRoot, manifest.bin.cperf),
+        'run',
+        '--database-url',
+        databaseUrl,
+        '--worker',
+        resolve(__dirname, 'fixtures/fake-worker.js'),
+        '--cases',
+        Buffer.from(JSON.stringify(cases)).toString('base64url'),
+        '--repetitions',
+        '1',
+        '--worker-timeout-ms',
+        '5000',
+        '--output',
+        output,
+      ]);
+      expect(result.error).toBeUndefined();
+      expect(result.status).toBe(0);
+      const report: BenchmarkReport = JSON.parse(
+        await readFile(output, 'utf8')
+      );
+      expect(report.validation.allRunsSucceeded).toBe(true);
+      expect(report.validation.freshProcessPerRun).toBe(true);
+      expect(report.summaries.baseline.sampleCount).toBe(1);
+      expect(JSON.stringify(report)).not.toContain(databaseUrl);
+      expect(result.stdout + result.stderr).not.toContain(databaseUrl);
+    } finally {
+      await rm(directory, { recursive: true, force: true });
+    }
+  });
+
+  test('reports CLI argument errors with a nonzero exit', () => {
+    const result = node([
+      resolve(artifactRoot, manifest.bin.cperf),
+      'prepare',
+      '--schema',
+      'cperf_example',
+    ]);
+    expect(result.error).toBeUndefined();
+    expect(result.status).toBe(1);
+    expect(result.stdout).toBe('');
+    expect(result.stderr).toContain('--database-url is required');
+  });
+});