Add a reusable Graphile performance harness

.github/workflows/run-tests.yaml

@@ -109,7 +109,7 @@ jobs:
           - batch: packages-core
             packages: 'packages/url-domains packages/coerce packages/csrf packages/oauth packages/12factor-env packages/orm packages/express-context packages/errors packages/llm-env packages/node-type-registry packages/query-spec packages/server-utils packages/site-deploy examples/site-deploy-ssg postgres/pg-cache postgres/pg-env'
           - batch: packages-services
-            packages: 'packages/postmaster packages/smtppostmaster packages/csv-to-pg packages/cli postgres/pgsql-client postgres/pg-ast'
+            packages: 'packages/postmaster packages/smtppostmaster packages/csv-to-pg packages/cli packages/perf-harness postgres/pgsql-client postgres/pg-ast'
           - batch: graphql
             packages: 'graphql/query graphql/codegen'
           - batch: graphile-unit

packages/perf-harness/README.md

@@ -0,0 +1,55 @@
+# Graphile performance harness
+
+Reusable infrastructure for measuring Graphile schema builds in fresh Node
+processes. The core accepts any list of serializable benchmark cases; it does not
+interpret case names or optimization-specific configuration.
+
+Each measurement receives a new PID, starts Node with `--expose-gc`, runs a
+deterministic GC sequence, records build time and memory metrics, validates a
+runtime query, and reports a schema hash. Cases can opt into schema equivalence
+groups and provide their own lifecycle validation through the worker result.
+
+## Extending the harness
+
+Define a suite and provide a dedicated worker entry point:
+
+```ts
+const suite = {
+  name: 'example',
+  cases: [
+    {
+      name: 'baseline',
+      workerConfig: { schemas: ['cperf_example'] },
+      expectedSchemaGroup: 'example-schema',
+    },
+  ],
+};
+
+await runBenchmarkSuite(suite, options, workerPath);
+```
+
+`workerConfig` must be JSON-serializable. Logic is implemented in the worker
+entry rather than serializing functions across process boundaries.
+
+Each worker has a five-minute wall-clock deadline, including startup, database
+access, validation, and cleanup. Set `options.workerTimeoutMs` or pass
+`--worker-timeout-ms` to `cperf run` to override it with a positive integer
+(maximum 2,147,483,647 ms). The effective deadline is recorded in `report.config`;
+it does not change the build-only `buildMs` measurement.
+
+A timed-out worker is killed and must finish closing before the next case starts.
+Its failure is recorded even if it already printed a successful result. If its
+process and stdio cannot be confirmed closed within another five seconds, the
+suite stops scheduling cases and writes a failed report containing the completed
+runs. Earlier successful samples may remain in summaries, but the report's
+validation fails. The CLI exits nonzero for either kind of failure.
+
+The package includes `stock-worker.js` as a minimal upstream Graphile baseline.
+The top-level commands require `--database-url`; the runner forwards it and the
+opaque case configuration to each short-lived worker as CLI arguments. Database
+credentials are redacted from worker failures and JSON reports. This harness is
+intended for local development on a trusted machine because command arguments
+may be visible to other local processes.
+
+The PostgreSQL fixture command only creates a previously absent schema whose
+name starts with `cperf_`; it never drops or replaces schemas.

packages/perf-harness/__tests__/fixture.test.ts

@@ -0,0 +1,233 @@
+import { Pool } from 'pg';
+
+import {
+  prepareFixture,
+  validateFixtureSchema,
+  validateFixtureTableCount,
+} from '../src/fixture';
+
+jest.mock('pg', () => ({
+  Pool: jest.fn(),
+}));
+
+type MockClient = {
+  query: jest.Mock;
+  release: jest.Mock;
+};
+
+type MockPool = {
+  connect: jest.Mock;
+  end: jest.Mock;
+};
+
+const fixtureOptions = {
+  databaseUrl: 'postgres://fixture-test',
+  schema: 'cperf_fixture_test',
+  tables: 1,
+};
+
+const queryResult = (text: string) => {
+  if (text.includes('pg_namespace')) {
+    return { rows: [{ exists: false }] };
+  }
+  if (text.includes('current_database')) {
+    return { rows: [{ database: 'fixture_db', server_version: '16' }] };
+  }
+  return { rows: [] };
+};
+
+const createMocks = (): { client: MockClient; pool: MockPool } => {
+  const client: MockClient = {
+    query: jest.fn(async (text: string) => queryResult(text)),
+    release: jest.fn(),
+  };
+  const pool: MockPool = {
+    connect: jest.fn().mockResolvedValue(client),
+    end: jest.fn().mockResolvedValue(undefined),
+  };
+  (Pool as unknown as jest.Mock).mockImplementation(() => pool);
+  return { client, pool };
+};
+
+const rejected = async (promise: Promise<unknown>): Promise<unknown> => {
+  try {
+    await promise;
+  } catch (error) {
+    return error;
+  }
+  throw new Error('expected prepareFixture to reject');
+};
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
+
+describe('fixture safety', () => {
+  test('only accepts narrowly scoped benchmark schema names', () => {
+    expect(validateFixtureSchema('cperf_example_1')).toBe('cperf_example_1');
+    expect(() => validateFixtureSchema('public')).toThrow(
+      'must start with cperf_'
+    );
+    expect(() =>
+      validateFixtureSchema('cperf_example; drop schema public')
+    ).toThrow('must start with cperf_');
+  });
+
+  test('bounds generated fixture size', () => {
+    expect(validateFixtureTableCount(64)).toBe(64);
+    expect(() => validateFixtureTableCount(0)).toThrow('between 1 and 500');
+    expect(() => validateFixtureTableCount(501)).toThrow('between 1 and 500');
+  });
+});
+
+describe('fixture resource lifecycle', () => {
+  test('ends the pool when connecting fails without releasing a client', async () => {
+    const { client, pool } = createMocks();
+    const connectError = new Error('connect failed');
+    pool.connect.mockRejectedValue(connectError);
+
+    await expect(prepareFixture(fixtureOptions)).rejects.toBe(connectError);
+
+    expect(client.release).not.toHaveBeenCalled();
+    expect(pool.end).toHaveBeenCalledTimes(1);
+  });
+
+  test('retains a falsy connection failure when ending the pool also fails', async () => {
+    const { client, pool } = createMocks();
+    const endError = new Error('end failed');
+    pool.connect.mockRejectedValue(undefined);
+    pool.end.mockRejectedValue(endError);
+
+    const error = await rejected(prepareFixture(fixtureOptions));
+
+    expect(error).toBeInstanceOf(AggregateError);
+    expect((error as AggregateError).errors).toEqual([undefined, endError]);
+    expect(
+      (error as AggregateError & { cause: unknown }).cause
+    ).toBeUndefined();
+    expect(client.release).not.toHaveBeenCalled();
+    expect(pool.end).toHaveBeenCalledTimes(1);
+  });
+
+  test('commits and releases the client before ending the pool on success', async () => {
+    const { client, pool } = createMocks();
+
+    await expect(prepareFixture(fixtureOptions)).resolves.toEqual(
+      expect.objectContaining({
+        database: 'fixture_db',
+        tableCount: 2,
+        functionCount: 1,
+      })
+    );
+
+    expect(client.query).toHaveBeenCalledWith('commit');
+    expect(client.release).toHaveBeenCalledTimes(1);
+    expect(pool.end).toHaveBeenCalledTimes(1);
+    expect(client.release.mock.invocationCallOrder[0]).toBeLessThan(
+      pool.end.mock.invocationCallOrder[0]
+    );
+  });
+
+  test('rolls back, releases, and ends resources after a SQL failure', async () => {
+    const { client, pool } = createMocks();
+    const queryError = new Error('create schema failed');
+    client.query.mockImplementation(async (text: string) => {
+      if (text.includes('create schema')) {
+        throw queryError;
+      }
+      return queryResult(text);
+    });
+
+    await expect(prepareFixture(fixtureOptions)).rejects.toBe(queryError);
+
+    expect(client.query).toHaveBeenCalledWith('rollback');
+    expect(client.release).toHaveBeenCalledTimes(1);
+    expect(pool.end).toHaveBeenCalledTimes(1);
+  });
+
+  test('retains a rollback failure with the original SQL failure', async () => {
+    const { client, pool } = createMocks();
+    const queryError = new Error('create schema failed');
+    const rollbackError = new Error('rollback failed');
+    client.query.mockImplementation(async (text: string) => {
+      if (text.includes('create schema')) {
+        throw queryError;
+      }
+      if (text === 'rollback') {
+        throw rollbackError;
+      }
+      return queryResult(text);
+    });
+
+    const error = await rejected(prepareFixture(fixtureOptions));
+
+    expect(error).toBeInstanceOf(AggregateError);
+    expect((error as AggregateError).errors).toEqual([
+      queryError,
+      rollbackError,
+    ]);
+    expect((error as AggregateError & { cause: unknown }).cause).toBe(
+      queryError
+    );
+    expect(client.release).toHaveBeenCalledTimes(1);
+    expect(pool.end).toHaveBeenCalledTimes(1);
+  });
+
+  test('still ends the pool when releasing the client fails', async () => {
+    const { client, pool } = createMocks();
+    const releaseError = new Error('release failed');
+    client.release.mockImplementation(() => {
+      throw releaseError;
+    });
+
+    await expect(prepareFixture(fixtureOptions)).rejects.toBe(releaseError);
+
+    expect(client.release).toHaveBeenCalledTimes(1);
+    expect(pool.end).toHaveBeenCalledTimes(1);
+  });
+
+  test('surfaces a pool end failure after releasing the client', async () => {
+    const { client, pool } = createMocks();
+    const endError = new Error('end failed');
+    pool.end.mockRejectedValue(endError);
+
+    await expect(prepareFixture(fixtureOptions)).rejects.toBe(endError);
+
+    expect(client.release).toHaveBeenCalledTimes(1);
+    expect(pool.end).toHaveBeenCalledTimes(1);
+  });
+
+  test('orders the primary and cleanup failures and preserves the primary cause', async () => {
+    const { client, pool } = createMocks();
+    const queryError = new Error('create schema failed');
+    const rollbackError = new Error('rollback failed');
+    const releaseError = new Error('release failed');
+    const endError = new Error('end failed');
+    client.query.mockImplementation(async (text: string) => {
+      if (text.includes('create schema')) {
+        throw queryError;
+      }
+      if (text === 'rollback') {
+        throw rollbackError;
+      }
+      return queryResult(text);
+    });
+    client.release.mockImplementation(() => {
+      throw releaseError;
+    });
+    pool.end.mockRejectedValue(endError);
+
+    const error = await rejected(prepareFixture(fixtureOptions));
+
+    expect(error).toBeInstanceOf(AggregateError);
+    expect((error as AggregateError).errors).toEqual([
+      queryError,
+      rollbackError,
+      releaseError,
+      endError,
+    ]);
+    expect((error as AggregateError & { cause: unknown }).cause).toBe(
+      queryError
+    );
+  });
+});

packages/perf-harness/__tests__/fixtures/fake-worker.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const valueFor = (name) => {
+  const flag = `--${name}`;
+  const index = process.argv.indexOf(flag);
+  if (index < 0 || !process.argv[index + 1]) {
+    throw new Error(`${flag} is required`);
+  }
+  return process.argv[index + 1];
+};
+
+const databaseUrl = valueFor('database-url');
+const envelope = JSON.parse(
+  Buffer.from(valueFor('worker-config'), 'base64url').toString('utf8')
+);
+if (envelope.workerConfig.pidFile) {
+  require('node:fs').writeFileSync(
+    envelope.workerConfig.pidFile,
+    String(process.pid)
+  );
+}
+const value = envelope.workerConfig.value;
+const memory = {
+  rss: value,
+  heapTotal: value,
+  heapUsed: value,
+  external: value,
+  arrayBuffers: value,
+};
+const result = {
+  status: 'ok',
+  pid: process.pid,
+  caseName: envelope.caseName,
+  buildMs: value,
+  schemaHash: envelope.workerConfig.schemaHash,
+  schemaTypeCount: 10,
+  runtimeVerified: true,
+  caseValidation: { passed: true, errors: [] },
+  memory: {
+    baseline: memory,
+    afterBuild: memory,
+    delta: memory,
+    processPeakRss: value,
+  },
+};
+if (envelope.workerConfig.mode !== 'hang') {
+  process.stdout.write(`CPERF_RESULT ${JSON.stringify(result)}\n`);
+}
+if (['hang', 'result-then-hang'].includes(envelope.workerConfig.mode)) {
+  process.stderr.write(`connecting to ${databaseUrl}\n`);
+  setInterval(() => undefined, 1_000);
+}