diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 7a89b17250..5a4b52fdf1 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -109,7 +109,7 @@ jobs: - batch: packages-core packages: 'packages/url-domains packages/coerce packages/csrf packages/oauth packages/12factor-env packages/orm packages/express-context packages/errors packages/llm-env packages/node-type-registry packages/query-spec packages/server-utils packages/site-deploy examples/site-deploy-ssg postgres/pg-cache postgres/pg-env' - batch: packages-services - packages: 'packages/postmaster packages/smtppostmaster packages/csv-to-pg packages/cli postgres/pgsql-client postgres/pg-ast' + packages: 'packages/postmaster packages/smtppostmaster packages/csv-to-pg packages/cli packages/perf-harness postgres/pgsql-client postgres/pg-ast' - batch: graphql packages: 'graphql/query graphql/codegen' - batch: graphile-unit diff --git a/graphql/server/src/middleware/__tests__/graphile-preset-composition.test.ts b/graphql/server/src/middleware/__tests__/graphile-preset-composition.test.ts new file mode 100644 index 0000000000..88370d4def --- /dev/null +++ b/graphql/server/src/middleware/__tests__/graphile-preset-composition.test.ts @@ -0,0 +1,342 @@ +import type { GraphileConfig } from 'graphile-config'; +import { resolvePreset } from 'graphile-config'; + +import { + assertGraphileCallerPresetsSafe, + composeGraphilePreset, + type ComposeGraphilePresetInput, + type GraphilePresetProtectionPolicy, +} from '../graphile-preset-composition'; + +const callerPlugin: GraphileConfig.Plugin = { + name: 'CallerPlugin', + version: '1.0.0', +}; + +const protectedPlugin: GraphileConfig.Plugin = { + name: 'ProtectedPlugin', + version: '1.0.0', +}; + +const exactService = { + name: 'main', + adaptor: 'constructive-test-adaptor', +} as unknown as NonNullable[number]; + +const protectedContext = jest.fn(() => ({ + pgSettings: { role: 'tenant_runtime' }, +})); +const protectedMaskError = jest.fn((error) => error); + +const protection: GraphilePresetProtectionPolicy = { + protectedPaths: ['pgServices', 'grafast.context', 'grafserv.maskError'], + protectedPluginNames: ['ProtectedPlugin'], +}; + +const compose = ( + overrides: Partial = {} +): GraphileConfig.Preset => + composeGraphilePreset({ + basePresets: [], + callerPresetsTrusted: true, + protection, + protectedPreset: { + plugins: [protectedPlugin], + pgServices: [exactService], + grafserv: { maskError: protectedMaskError }, + grafast: { context: protectedContext }, + }, + ...overrides, + }); + +const captureError = (callback: () => unknown): unknown => { + try { + callback(); + } catch (error) { + return error; + } + throw new Error('Expected callback to throw'); +}; + +describe('Graphile caller preset composition', () => { + it.each([ + { callerExtends: [{ plugins: [callerPlugin] }] }, + { callerPreset: { schema: { defaultBehavior: '-delete' } } }, + ])( + 'rejects non-empty caller code until it is explicitly trusted', + (caller) => { + const getter = jest.fn(() => ({ context: protectedContext })); + const callerPreset = caller.callerPreset ?? {}; + Object.defineProperty(callerPreset, 'unrelatedAccessor', { + enumerable: true, + get: getter, + }); + + expect(() => + compose({ + ...caller, + callerPreset, + callerPresetsTrusted: false, + }) + ).toThrow( + expect.objectContaining({ code: 'GRAPHILE_CALLER_PRESET_NOT_TRUSTED' }) + ); + expect(getter).not.toHaveBeenCalled(); + } + ); + + it('allows empty defaults without widening the trust boundary', () => { + expect(() => + compose({ + callerExtends: [], + callerPreset: {}, + callerPresetsTrusted: false, + }) + ).not.toThrow(); + }); + + it('applies layers in deterministic base, caller, protected order', () => { + const basePreset: GraphileConfig.Preset = { schema: {} }; + const callerExtension: GraphileConfig.Preset = { grafserv: {} }; + const callerPreset: GraphileConfig.Preset = { grafast: {} }; + const protectedExtension: GraphileConfig.Preset = { schema: {} }; + const protectedRootExtension: GraphileConfig.Preset = { grafserv: {} }; + + const result = compose({ + basePresets: [basePreset], + callerExtends: [callerExtension], + callerPreset, + protectedPresets: [protectedExtension], + protectedPreset: { + extends: [protectedRootExtension], + plugins: [protectedPlugin], + }, + }); + + expect(result).toEqual({ + extends: [ + basePreset, + callerExtension, + callerPreset, + protectedExtension, + protectedRootExtension, + ], + plugins: [protectedPlugin], + }); + }); + + it('allows caller plugins and unprotected scope settings', () => { + const resolved = resolvePreset( + compose({ + callerPreset: { + plugins: [callerPlugin], + schema: { defaultBehavior: '-delete' }, + grafserv: { maxRequestLength: 123_456 }, + }, + }) + ); + + expect(resolved.plugins).toEqual( + expect.arrayContaining([callerPlugin, protectedPlugin]) + ); + expect(resolved.schema).toMatchObject({ defaultBehavior: '-delete' }); + expect(resolved.grafserv).toMatchObject({ + maxRequestLength: 123_456, + maskError: protectedMaskError, + }); + expect(resolved.grafast).toMatchObject({ context: protectedContext }); + expect(resolved.pgServices).toEqual([exactService]); + }); + + it.each([ + [{ pgServices: [{ name: 'other' }] }, 'pgServices'], + [{ grafast: { context: () => ({}) } }, 'grafast.context'], + [ + { grafserv: { maskError: (error: unknown) => error } }, + 'grafserv.maskError', + ], + ])( + 'rejects caller ownership of protected paths', + (callerPreset, protectedSetting) => { + const error = captureError(() => + compose({ + callerPreset: callerPreset as unknown as GraphileConfig.Preset, + }) + ); + + expect(error).toMatchObject({ + code: 'GRAPHILE_PROTECTED_PRESET_OVERRIDE', + context: { + presetPath: 'graphile.preset', + protectedSetting, + }, + }); + } + ); + + it('rejects protected paths hidden behind accessors without invoking them', () => { + const getter = jest.fn(() => ({ context: protectedContext })); + const callerPreset: GraphileConfig.Preset = {}; + Object.defineProperty(callerPreset, 'grafast', { + enumerable: true, + get: getter, + }); + + expect(() => compose({ callerPreset })).toThrow( + expect.objectContaining({ + code: 'GRAPHILE_PROTECTED_PRESET_OVERRIDE', + context: expect.objectContaining({ + protectedSetting: 'grafast.context', + }), + }) + ); + expect(getter).not.toHaveBeenCalled(); + }); + + it.each([ + [{ plugins: [protectedPlugin] }, 'plugins.ProtectedPlugin'], + [{ disablePlugins: ['ProtectedPlugin'] }, 'disablePlugins.ProtectedPlugin'], + ])( + 'rejects protected plugin replacement and disablement', + (callerPreset, protectedSetting) => { + const error = captureError(() => + compose({ callerPreset: callerPreset as GraphileConfig.Preset }) + ); + + expect(error).toMatchObject({ + code: 'GRAPHILE_PROTECTED_PRESET_OVERRIDE', + context: { + presetPath: 'graphile.preset', + protectedSetting, + }, + }); + } + ); + + it('rejects protected settings in nested extends with their exact path', () => { + const error = captureError(() => + compose({ + callerExtends: [ + { + extends: [ + { + grafserv: { maskError: protectedMaskError }, + }, + ], + }, + ], + }) + ); + + expect(error).toMatchObject({ + code: 'GRAPHILE_PROTECTED_PRESET_OVERRIDE', + context: { + presetPath: 'graphile.extends[0].extends[0]', + protectedSetting: 'grafserv.maskError', + }, + }); + }); + + it('rejects circular caller preset graphs deterministically', () => { + const cyclic: GraphileConfig.Preset = {}; + cyclic.extends = [cyclic]; + + const error = captureError(() => + assertGraphileCallerPresetsSafe( + { + callerExtends: [cyclic], + callerPresetsTrusted: true, + }, + protection + ) + ); + + expect(error).toMatchObject({ + code: 'GRAPHILE_CALLER_PRESET_INVALID', + context: { + presetPath: 'graphile.extends[0].extends[0]', + reason: 'extends must not contain a cycle', + }, + }); + }); + + it('does not admit configuration inherited through a preset prototype', () => { + const callerPreset = Object.create({ + extends: [{ pgServices: [{ name: 'other' }] }], + }) as GraphileConfig.Preset; + + expect(() => + compose({ callerPreset, callerPresetsTrusted: false }) + ).toThrow( + expect.objectContaining({ code: 'GRAPHILE_CALLER_PRESET_NOT_TRUSTED' }) + ); + + expect(() => compose({ callerPreset })).toThrow( + expect.objectContaining({ + code: 'GRAPHILE_CALLER_PRESET_INVALID', + context: expect.objectContaining({ + presetPath: 'graphile.preset', + reason: 'preset must be a plain object', + }), + }) + ); + }); + + it.each([ + [{ extends: {} }, 'extends must be an array'], + [{ plugins: {} }, 'plugins must be an array'], + [ + { disablePlugins: [protectedPlugin] }, + 'disabled plugin name must be a string', + ], + ])('rejects malformed caller preset fields', (callerPreset, reason) => { + const error = captureError(() => + compose({ + callerPreset: callerPreset as unknown as GraphileConfig.Preset, + }) + ); + + expect(error).toMatchObject({ + code: 'GRAPHILE_CALLER_PRESET_INVALID', + context: expect.objectContaining({ reason }), + }); + }); + + it('accepts a shared nested preset that is not circular', () => { + const shared: GraphileConfig.Preset = { + schema: { defaultBehavior: '-delete' }, + }; + + expect(() => + compose({ + callerExtends: [{ extends: [shared] }, { extends: [shared] }], + }) + ).not.toThrow(); + }); + + it('accepts future feature-owned protection without changing the primitive', () => { + const futureProtection: GraphilePresetProtectionPolicy = { + protectedPaths: ['schema.futureSecuritySetting'], + protectedPluginNames: ['FutureAdmissionPlugin'], + }; + + expect(() => + assertGraphileCallerPresetsSafe( + { + callerPreset: { + schema: { futureSecuritySetting: false }, + } as unknown as GraphileConfig.Preset, + callerPresetsTrusted: true, + }, + futureProtection + ) + ).toThrow( + expect.objectContaining({ + code: 'GRAPHILE_PROTECTED_PRESET_OVERRIDE', + context: expect.objectContaining({ + protectedSetting: 'schema.futureSecuritySetting', + }), + }) + ); + }); +}); diff --git a/graphql/server/src/middleware/graphile-preset-composition.ts b/graphql/server/src/middleware/graphile-preset-composition.ts new file mode 100644 index 0000000000..b09a8a55b2 --- /dev/null +++ b/graphql/server/src/middleware/graphile-preset-composition.ts @@ -0,0 +1,285 @@ +import { errors } from '@constructive-io/errors'; +import type { GraphileConfig } from 'graphile-config'; + +type PresetRecord = Record; + +export interface GraphilePresetProtectionPolicy { + /** Dot-separated preset fields that caller configuration may not own. */ + protectedPaths: readonly string[]; + /** Plugin names that callers may neither register nor disable. */ + protectedPluginNames: readonly string[]; +} + +export interface GraphileCallerPresetInput { + callerExtends?: readonly GraphileConfig.Preset[]; + callerPreset?: Partial; + /** Whether all caller preset code has been admitted into the process TCB. */ + callerPresetsTrusted: boolean; +} + +export interface ComposeGraphilePresetInput extends GraphileCallerPresetInput { + /** CNC defaults that trusted callers may customize. */ + basePresets: readonly GraphileConfig.Preset[]; + /** CNC-owned presets resolved after caller customization. */ + protectedPresets?: readonly GraphileConfig.Preset[]; + /** CNC-owned root fields that retain final precedence. */ + protectedPreset: GraphileConfig.Preset; + protection: GraphilePresetProtectionPolicy; +} + +const isObjectRecord = (value: unknown): value is PresetRecord => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const isPlainObjectRecord = (value: unknown): value is PresetRecord => { + if (!isObjectRecord(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +}; + +const invalidCallerPreset = (presetPath: string, reason: string): never => { + throw errors.GRAPHILE_CALLER_PRESET_INVALID({ presetPath, reason }); +}; + +const rejectProtectedOverride = ( + presetPath: string, + protectedSetting: string +): never => { + throw errors.GRAPHILE_PROTECTED_PRESET_OVERRIDE({ + presetPath, + protectedSetting, + }); +}; + +const readOwnDataProperty = ( + record: PresetRecord, + field: string, + presetPath: string +): { present: false } | { present: true; value: unknown } => { + const descriptor = Object.getOwnPropertyDescriptor(record, field); + if (!descriptor) return { present: false }; + if (!('value' in descriptor)) { + return invalidCallerPreset( + presetPath, + `${field} must be declared as a data property` + ); + } + return { present: true, value: descriptor.value }; +}; + +/** + * Accessors and non-object intermediate values count as overrides. They could + * otherwise hide a protected value from validation and reveal it at resolve + * time. + */ +const ownsProtectedPath = ( + preset: PresetRecord, + pathSegments: readonly string[] +): boolean => { + let current = preset; + for (let index = 0; index < pathSegments.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor( + current, + pathSegments[index] + ); + if (!descriptor) return false; + if (!('value' in descriptor)) return true; + if (index === pathSegments.length - 1) return true; + if (!isObjectRecord(descriptor.value)) return true; + current = descriptor.value; + } + return false; +}; + +const assertProtectedPathsAreNotOwned = ( + preset: PresetRecord, + presetPath: string, + protectedPaths: readonly string[] +): void => { + for (const protectedSetting of protectedPaths) { + const pathSegments = protectedSetting.split('.'); + if (ownsProtectedPath(preset, pathSegments)) { + rejectProtectedOverride(presetPath, protectedSetting); + } + } +}; + +const assertProtectedPluginsAreNotOwned = ( + preset: PresetRecord, + presetPath: string, + protectedPluginNames: ReadonlySet +): void => { + const pluginsProperty = readOwnDataProperty(preset, 'plugins', presetPath); + if (pluginsProperty.present) { + const plugins = pluginsProperty.value; + if (!Array.isArray(plugins)) { + return invalidCallerPreset(presetPath, 'plugins must be an array'); + } + plugins.forEach((plugin, index) => { + const pluginPath = `${presetPath}.plugins[${index}]`; + if (!isPlainObjectRecord(plugin)) { + return invalidCallerPreset(pluginPath, 'plugin must be an object'); + } + const nameProperty = readOwnDataProperty(plugin, 'name', pluginPath); + const pluginName = nameProperty.present ? nameProperty.value : undefined; + if (typeof pluginName !== 'string') { + return invalidCallerPreset(pluginPath, 'plugin name must be a string'); + } + if (protectedPluginNames.has(pluginName)) { + rejectProtectedOverride(presetPath, `plugins.${pluginName}`); + } + }); + } + + const disabledProperty = readOwnDataProperty( + preset, + 'disablePlugins', + presetPath + ); + if (disabledProperty.present) { + const disabledPlugins = disabledProperty.value; + if (!Array.isArray(disabledPlugins)) { + return invalidCallerPreset(presetPath, 'disablePlugins must be an array'); + } + disabledPlugins.forEach((pluginName, index) => { + if (typeof pluginName !== 'string') { + return invalidCallerPreset( + `${presetPath}.disablePlugins[${index}]`, + 'disabled plugin name must be a string' + ); + } + if (protectedPluginNames.has(pluginName)) { + rejectProtectedOverride(presetPath, `disablePlugins.${pluginName}`); + } + }); + } +}; + +const assertPresetDoesNotOverrideProtectedSettings = ( + preset: unknown, + presetPath: string, + protection: GraphilePresetProtectionPolicy, + protectedPluginNames: ReadonlySet, + visiting: Set, + validated: Set +): void => { + if (!isPlainObjectRecord(preset)) { + return invalidCallerPreset(presetPath, 'preset must be a plain object'); + } + const presetRecord = preset as PresetRecord; + if (visiting.has(presetRecord)) { + invalidCallerPreset(presetPath, 'extends must not contain a cycle'); + } + if (validated.has(presetRecord)) return; + + visiting.add(presetRecord); + assertProtectedPathsAreNotOwned( + presetRecord, + presetPath, + protection.protectedPaths + ); + assertProtectedPluginsAreNotOwned( + presetRecord, + presetPath, + protectedPluginNames + ); + + const extendsProperty = readOwnDataProperty( + presetRecord, + 'extends', + presetPath + ); + if (extendsProperty.present) { + const extendedPresets = extendsProperty.value; + if (!Array.isArray(extendedPresets)) { + return invalidCallerPreset(presetPath, 'extends must be an array'); + } + extendedPresets.forEach((nestedPreset, index) => { + assertPresetDoesNotOverrideProtectedSettings( + nestedPreset, + `${presetPath}.extends[${index}]`, + protection, + protectedPluginNames, + visiting, + validated + ); + }); + } + + visiting.delete(presetRecord); + validated.add(presetRecord); +}; + +const hasCallerPresetConfiguration = ( + input: GraphileCallerPresetInput +): boolean => { + if (input.callerExtends !== undefined) { + if (!Array.isArray(input.callerExtends)) return true; + if (input.callerExtends.length > 0) return true; + } + if (input.callerPreset === undefined) return false; + if (!isPlainObjectRecord(input.callerPreset)) return true; + return Reflect.ownKeys(input.callerPreset).length > 0; +}; + +/** Validate caller code before it enters Graphile's preset resolver. */ +export const assertGraphileCallerPresetsSafe = ( + input: GraphileCallerPresetInput, + protection: GraphilePresetProtectionPolicy +): void => { + if (!input.callerPresetsTrusted && hasCallerPresetConfiguration(input)) { + throw errors.GRAPHILE_CALLER_PRESET_NOT_TRUSTED(); + } + + const callerExtends = input.callerExtends ?? []; + if (!Array.isArray(callerExtends)) { + invalidCallerPreset('graphile.extends', 'value must be an array'); + } + + const protectedPluginNames = new Set(protection.protectedPluginNames); + const validated = new Set(); + callerExtends.forEach((preset, index) => { + assertPresetDoesNotOverrideProtectedSettings( + preset, + `graphile.extends[${index}]`, + protection, + protectedPluginNames, + new Set(), + validated + ); + }); + if (input.callerPreset !== undefined) { + assertPresetDoesNotOverrideProtectedSettings( + input.callerPreset, + 'graphile.preset', + protection, + protectedPluginNames, + new Set(), + validated + ); + } +}; + +/** + * Compose Graphile configuration in deterministic trust order. CNC-owned root + * fields are emitted last and therefore keep final precedence. + */ +export const composeGraphilePreset = ( + input: ComposeGraphilePresetInput +): GraphileConfig.Preset => { + assertGraphileCallerPresetsSafe(input, input.protection); + + const callerExtends = input.callerExtends ?? []; + const { extends: protectedRootExtends = [], ...protectedRoot } = + input.protectedPreset; + + return { + extends: [ + ...input.basePresets, + ...callerExtends, + ...(input.callerPreset ? [input.callerPreset] : []), + ...(input.protectedPresets ?? []), + ...protectedRootExtends, + ], + ...protectedRoot, + }; +}; diff --git a/graphql/server/src/middleware/graphile.ts b/graphql/server/src/middleware/graphile.ts index dca05e19c9..0aa30d94ae 100644 --- a/graphql/server/src/middleware/graphile.ts +++ b/graphql/server/src/middleware/graphile.ts @@ -21,6 +21,10 @@ import { AuthCookiePlugin } from '../plugins/auth-cookie-plugin'; import { createErrorEventsPlugin } from '../plugins/error-events-plugin'; import { RequestProtectionPlugin } from '../plugins/request-protection-plugin'; import type { DatabaseSettings } from '../types'; +import { + composeGraphilePreset, + type GraphilePresetProtectionPolicy +} from './graphile-preset-composition'; import { maskError } from './mask-error'; import { observeGraphileBuild } from './observability/graphile-build-stats'; @@ -63,6 +67,22 @@ export function clearInFlightMap(): void { const log = new Logger('graphile'); const reqLabel = (req: Request): string => (req.requestId ? `[${req.requestId}]` : '[req]'); +// Protect only fields and plugins currently owned by this server. Future +// features extend this policy alongside their own activation. +const GRAPHILE_PRESET_PROTECTION = { + protectedPaths: [ + 'pgServices', + 'grafast.context', + 'grafast.explain', + 'grafserv.graphqlPath', + 'grafserv.graphiqlPath', + 'grafserv.graphiql', + 'grafserv.graphiqlOnGraphQLGET', + 'grafserv.maskError' + ], + protectedPluginNames: ['AuthCookiePlugin', 'FunctionBindingsPlugin'] +} as const satisfies GraphilePresetProtectionPolicy; + /** * Build a PostGraphile v5 preset for a tenant. * @@ -81,8 +101,7 @@ const buildPreset = ( apiId?: string, compute?: ComputeConfig ): GraphileConfig.Preset => { - return { - extends: [createConstructivePreset(databaseSettings)], + const protectedPreset: GraphileConfig.Preset = { plugins: [ AuthCookiePlugin, RequestProtectionPlugin, @@ -272,6 +291,15 @@ const buildPreset = ( } } }; + + return composeGraphilePreset({ + basePresets: [createConstructivePreset(databaseSettings)], + // Caller configuration remains intentionally dormant until build/cache + // identity includes preset composition (F17). + callerPresetsTrusted: false, + protection: GRAPHILE_PRESET_PROTECTION, + protectedPreset + }); }; export const graphile = (opts: ConstructiveOptions): RequestHandler => { diff --git a/packages/errors/__tests__/graphile-preset-errors.test.ts b/packages/errors/__tests__/graphile-preset-errors.test.ts new file mode 100644 index 0000000000..789bef9662 --- /dev/null +++ b/packages/errors/__tests__/graphile-preset-errors.test.ts @@ -0,0 +1,42 @@ +import { classify, errors } from '../src'; + +describe('Graphile preset configuration errors', () => { + it('classifies caller trust failures as internal startup errors', () => { + const error = errors.GRAPHILE_CALLER_PRESET_NOT_TRUSTED(); + + expect(error).toMatchObject({ + code: 'GRAPHILE_CALLER_PRESET_NOT_TRUSTED', + errorClass: 'internal', + http: 500, + }); + expect(classify(error.code)).toBe('internal'); + }); + + it('retains safe context for malformed caller presets', () => { + const error = errors.GRAPHILE_CALLER_PRESET_INVALID({ + presetPath: 'graphile.extends[0]', + reason: 'extends must not contain a cycle', + }); + + expect(error.code).toBe('GRAPHILE_CALLER_PRESET_INVALID'); + expect(error.context).toEqual({ + presetPath: 'graphile.extends[0]', + reason: 'extends must not contain a cycle', + }); + expect(error.message).toContain('graphile.extends[0]'); + }); + + it('identifies the protected setting without including its value', () => { + const error = errors.GRAPHILE_PROTECTED_PRESET_OVERRIDE({ + presetPath: 'graphile.preset', + protectedSetting: 'grafast.context', + }); + + expect(error.code).toBe('GRAPHILE_PROTECTED_PRESET_OVERRIDE'); + expect(error.context).toEqual({ + presetPath: 'graphile.preset', + protectedSetting: 'grafast.context', + }); + expect(error.message).not.toContain('pgSettings'); + }); +}); diff --git a/packages/errors/src/registry.ts b/packages/errors/src/registry.ts index ec4e7f005e..4a377f3bba 100644 --- a/packages/errors/src/registry.ts +++ b/packages/errors/src/registry.ts @@ -413,6 +413,32 @@ export const registry = { message: 'A value conflicts with an existing record.' }), + // =========================================================================== + // Graphile startup configuration (internal) + // =========================================================================== + GRAPHILE_CALLER_PRESET_NOT_TRUSTED: defineError({ + code: 'GRAPHILE_CALLER_PRESET_NOT_TRUSTED', + class: 'internal', + http: 500, + message: 'Graphile caller presets have not been admitted into the server trust boundary.' + }), + GRAPHILE_CALLER_PRESET_INVALID: defineError<{ presetPath: string; reason: string }>({ + code: 'GRAPHILE_CALLER_PRESET_INVALID', + class: 'internal', + http: 500, + message: 'Graphile caller preset "{{presetPath}}" is invalid: {{reason}}.' + }), + GRAPHILE_PROTECTED_PRESET_OVERRIDE: defineError<{ + presetPath: string; + protectedSetting: string; + }>({ + code: 'GRAPHILE_PROTECTED_PRESET_OVERRIDE', + class: 'internal', + http: 500, + message: + 'Graphile caller preset "{{presetPath}}" may not configure protected setting "{{protectedSetting}}".' + }), + // =========================================================================== // pgpm CLI / engine (mostly internal) — behavior preserved from the former // pgpm/types error-factory so existing call sites are unchanged. diff --git a/packages/perf-harness/README.md b/packages/perf-harness/README.md new file mode 100644 index 0000000000..01fc2f1d47 --- /dev/null +++ b/packages/perf-harness/README.md @@ -0,0 +1,42 @@ +# Graphile performance harness + +Reusable infrastructure for measuring Graphile schema builds in fresh Node +processes. The core accepts any list of serializable benchmark cases; it does not +interpret case names or optimization-specific configuration. + +Each measurement receives a new PID, starts Node with `--expose-gc`, runs a +deterministic GC sequence, records build time and memory metrics, validates a +runtime query, and reports a schema hash. Cases can opt into schema equivalence +groups and provide their own lifecycle validation through the worker result. + +## Extending the harness + +Define a suite and provide a dedicated worker entry point: + +```ts +const suite = { + name: 'example', + cases: [ + { + name: 'baseline', + workerConfig: { schemas: ['cperf_example'] }, + expectedSchemaGroup: 'example-schema', + }, + ], +}; + +await runBenchmarkSuite(suite, options, workerPath); +``` + +`workerConfig` must be JSON-serializable. Logic is implemented in the worker +entry rather than serializing functions across process boundaries. + +The package includes `stock-worker.js` as a minimal upstream Graphile baseline. +The top-level commands require `--database-url`; the runner forwards it and the +opaque case configuration to each short-lived worker as CLI arguments. Database +credentials are redacted from worker failures and JSON reports. This harness is +intended for local development on a trusted machine because command arguments +may be visible to other local processes. + +The PostgreSQL fixture command only creates a previously absent schema whose +name starts with `cperf_`; it never drops or replaces schemas. diff --git a/packages/perf-harness/__tests__/fixture.test.ts b/packages/perf-harness/__tests__/fixture.test.ts new file mode 100644 index 0000000000..08e9109e1e --- /dev/null +++ b/packages/perf-harness/__tests__/fixture.test.ts @@ -0,0 +1,22 @@ +import { + validateFixtureSchema, + validateFixtureTableCount, +} from '../src/fixture'; + +describe('fixture safety', () => { + test('only accepts narrowly scoped benchmark schema names', () => { + expect(validateFixtureSchema('cperf_example_1')).toBe('cperf_example_1'); + expect(() => validateFixtureSchema('public')).toThrow( + 'must start with cperf_' + ); + expect(() => + validateFixtureSchema('cperf_example; drop schema public') + ).toThrow('must start with cperf_'); + }); + + test('bounds generated fixture size', () => { + expect(validateFixtureTableCount(64)).toBe(64); + expect(() => validateFixtureTableCount(0)).toThrow('between 1 and 500'); + expect(() => validateFixtureTableCount(501)).toThrow('between 1 and 500'); + }); +}); diff --git a/packages/perf-harness/__tests__/fixtures/fake-worker.js b/packages/perf-harness/__tests__/fixtures/fake-worker.js new file mode 100644 index 0000000000..5ba1e05b83 --- /dev/null +++ b/packages/perf-harness/__tests__/fixtures/fake-worker.js @@ -0,0 +1,40 @@ +'use strict'; + +const valueFor = (name) => { + const flag = `--${name}`; + const index = process.argv.indexOf(flag); + if (index < 0 || !process.argv[index + 1]) { + throw new Error(`${flag} is required`); + } + return process.argv[index + 1]; +}; + +valueFor('database-url'); +const envelope = JSON.parse( + Buffer.from(valueFor('worker-config'), 'base64url').toString('utf8') +); +const value = envelope.workerConfig.value; +const memory = { + rss: value, + heapTotal: value, + heapUsed: value, + external: value, + arrayBuffers: value, +}; +const result = { + status: 'ok', + pid: process.pid, + caseName: envelope.caseName, + buildMs: value, + schemaHash: envelope.workerConfig.schemaHash, + schemaTypeCount: 10, + runtimeVerified: true, + caseValidation: { passed: true, errors: [] }, + memory: { + baseline: memory, + afterBuild: memory, + delta: memory, + processPeakRss: value, + }, +}; +process.stdout.write(`CPERF_RESULT ${JSON.stringify(result)}\n`); diff --git a/packages/perf-harness/__tests__/process.test.ts b/packages/perf-harness/__tests__/process.test.ts new file mode 100644 index 0000000000..4016c8df51 --- /dev/null +++ b/packages/perf-harness/__tests__/process.test.ts @@ -0,0 +1,71 @@ +import { resolve } from 'node:path'; + +import { parseWorkerProcessArgs, runWorkerProcess } from '../src/process'; + +describe('worker CLI protocol', () => { + const encodedConfig = Buffer.from( + JSON.stringify({ caseName: 'baseline', workerConfig: { value: 1 } }) + ).toString('base64url'); + + test('parses the database URL and worker envelope from CLI arguments', () => { + expect( + parseWorkerProcessArgs([ + '--database-url', + 'postgres:///benchmark', + '--worker-config', + encodedConfig, + ]) + ).toEqual({ + databaseUrl: 'postgres:///benchmark', + envelope: { caseName: 'baseline', workerConfig: { value: 1 } }, + }); + }); + + test('rejects missing, duplicate, and unsupported worker arguments', () => { + expect(() => + parseWorkerProcessArgs(['--worker-config', encodedConfig]) + ).toThrow('--database-url is required'); + expect(() => + parseWorkerProcessArgs(['--database-url', 'postgres:///benchmark']) + ).toThrow('--worker-config is required'); + expect(() => + parseWorkerProcessArgs([ + '--database-url', + 'postgres:///one', + '--database-url', + 'postgres:///two', + '--worker-config', + encodedConfig, + ]) + ).toThrow('--database-url may only be specified once'); + expect(() => + parseWorkerProcessArgs([ + '--database-url', + 'postgres:///benchmark', + '--worker-config', + encodedConfig, + '--unexpected', + 'value', + ]) + ).toThrow("unsupported worker argument '--unexpected'"); + }); +}); + +describe('fresh worker process', () => { + test('uses distinct PIDs and does not expose the database URL', async () => { + const worker = resolve(__dirname, 'fixtures/fake-worker.js'); + const definition = { + name: 'baseline', + workerConfig: { value: 1, schemaHash: 'same' }, + }; + const databaseUrl = 'postgres://secret@example.test/database'; + const first = await runWorkerProcess(worker, databaseUrl, definition); + const second = await runWorkerProcess(worker, databaseUrl, definition); + expect(first.pid).not.toBe(process.pid); + expect(second.pid).not.toBe(process.pid); + expect(first.pid).not.toBe(second.pid); + expect(JSON.stringify([first.result, second.result])).not.toContain( + databaseUrl + ); + }); +}); diff --git a/packages/perf-harness/__tests__/report.test.ts b/packages/perf-harness/__tests__/report.test.ts new file mode 100644 index 0000000000..6d7e043d7a --- /dev/null +++ b/packages/perf-harness/__tests__/report.test.ts @@ -0,0 +1,78 @@ +import { + compareCases, + summarizeCase, + validateSchemaGroups, +} from '../src/report'; +import type { BenchmarkRun, SuccessfulWorkerResult } from '../src/types'; + +const result = (caseName: string, value: number): SuccessfulWorkerResult => ({ + status: 'ok', + pid: value, + caseName, + buildMs: value, + schemaHash: 'same', + schemaTypeCount: 10, + runtimeVerified: true, + caseValidation: { passed: true, errors: [] }, + memory: { + baseline: { + rss: 10, + heapTotal: 10, + heapUsed: 10, + external: 10, + arrayBuffers: 10, + }, + afterBuild: { + rss: value, + heapTotal: value, + heapUsed: value, + external: value, + arrayBuffers: value, + }, + delta: { + rss: value - 10, + heapTotal: value - 10, + heapUsed: value - 10, + external: value - 10, + arrayBuffers: value - 10, + }, + processPeakRss: value, + }, +}); + +describe('generic reports', () => { + test('summarizes arbitrary cases and compares medians', () => { + const runs: BenchmarkRun[] = [10, 30, 20].map((value, index) => ({ + repetition: index + 1, + position: 1, + caseName: 'base', + result: result('base', value), + })); + runs.push( + ...[5, 15, 10].map((value, index) => ({ + repetition: index + 1, + position: 2, + caseName: 'candidate', + result: result('candidate', value), + })) + ); + const base = summarizeCase(runs, 'base')!; + const candidate = summarizeCase(runs, 'candidate')!; + expect( + compareCases('base', 'candidate', base, candidate).buildMs.percentChange + ).toBe(-50); + expect( + validateSchemaGroups( + [ + { name: 'base', workerConfig: null, expectedSchemaGroup: 'schema' }, + { + name: 'candidate', + workerConfig: null, + expectedSchemaGroup: 'schema', + }, + ], + runs + ) + ).toEqual({ equivalent: true, hashes: { schema: 'same' }, errors: [] }); + }); +}); diff --git a/packages/perf-harness/__tests__/run.test.ts b/packages/perf-harness/__tests__/run.test.ts new file mode 100644 index 0000000000..4e8ef65d93 --- /dev/null +++ b/packages/perf-harness/__tests__/run.test.ts @@ -0,0 +1,43 @@ +import { resolve } from 'node:path'; + +import { cliMain, runBenchmarkSuite } from '../src/run'; + +describe('generic suite runner', () => { + test('requires the database URL as an explicit CLI argument', async () => { + await expect( + cliMain(['prepare', '--schema', 'cperf_explicit_cli']) + ).rejects.toThrow('--database-url is required'); + }); + + test('validates fresh processes and schema groups without fixed case names', async () => { + const report = await runBenchmarkSuite( + { + name: 'test-suite', + cases: ['alpha', 'beta', 'gamma'].map((name, index) => ({ + name, + workerConfig: { value: index + 1, schemaHash: 'same' }, + expectedSchemaGroup: 'schema', + })), + }, + { + databaseUrl: 'postgres:///not-used-by-fake-worker', + repetitions: 1, + seed: 1, + order: ['alpha', 'beta', 'gamma'], + }, + resolve(__dirname, 'fixtures/fake-worker.js') + ); + expect(report.validation).toEqual( + expect.objectContaining({ + allRunsSucceeded: true, + freshProcessPerRun: true, + caseValidationPassed: true, + schemaGroupsEquivalent: true, + schemaGroups: { schema: 'same' }, + errors: [], + }) + ); + expect(new Set(report.runs.map((run) => run.result.pid)).size).toBe(3); + expect(JSON.stringify(report)).not.toContain('postgres:///'); + }); +}); diff --git a/packages/perf-harness/__tests__/schedule.test.ts b/packages/perf-harness/__tests__/schedule.test.ts new file mode 100644 index 0000000000..89d5e672f8 --- /dev/null +++ b/packages/perf-harness/__tests__/schedule.test.ts @@ -0,0 +1,29 @@ +import { makeSchedule } from '../src/schedule'; + +const cases = [ + { name: 'a', workerConfig: null }, + { name: 'b', workerConfig: null }, + { name: 'c', workerConfig: null }, +]; + +describe('generic benchmark scheduling', () => { + test('is deterministic and supports any case list', () => { + const first = makeSchedule(cases, 4, 1234); + expect(makeSchedule(cases, 4, 1234)).toEqual(first); + expect(makeSchedule(cases, 4, 4321)).not.toEqual(first); + for (let repetition = 1; repetition <= 4; repetition += 1) { + expect( + first + .filter((item) => item.repetition === repetition) + .map((item) => item.caseName) + .sort() + ).toEqual(['a', 'b', 'c']); + } + }); + + test('accepts an exact order containing each case once', () => { + expect( + makeSchedule(cases, 2, 1, ['c', 'a', 'b']).map((item) => item.caseName) + ).toEqual(['c', 'a', 'b', 'c', 'a', 'b']); + }); +}); diff --git a/packages/perf-harness/jest.config.js b/packages/perf-harness/jest.config.js new file mode 100644 index 0000000000..f34711d4ed --- /dev/null +++ b/packages/perf-harness/jest.config.js @@ -0,0 +1,12 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': ['ts-jest', { tsconfig: 'tsconfig.json' }], + }, + testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], + modulePathIgnorePatterns: ['dist/*'], + testPathIgnorePatterns: ['/__tests__/fixtures/'], +}; diff --git a/packages/perf-harness/package.json b/packages/perf-harness/package.json new file mode 100644 index 0000000000..d975054937 --- /dev/null +++ b/packages/perf-harness/package.json @@ -0,0 +1,36 @@ +{ + "name": "@constructive-io/perf-harness", + "version": "0.1.0", + "private": true, + "description": "Reusable fresh-process Graphile performance harness", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "bin": { + "cperf": "index.js" + }, + "scripts": { + "clean": "makage clean", + "build": "makage build", + "build:dev": "makage build --dev", + "lint": "eslint . --fix", + "test": "jest" + }, + "dependencies": { + "graphile-build": "5.1.1", + "graphile-build-pg": "5.1.3", + "graphile-config": "1.1.0", + "graphql": "16.13.0", + "pg": "^8.21.0", + "postgraphile": "5.1.4" + }, + "devDependencies": { + "@types/node": "^22.19.11", + "@types/pg": "^8.20.4", + "makage": "^0.8.0" + }, + "engines": { + "node": ">=22" + }, + "license": "MIT" +} diff --git a/packages/perf-harness/src/fixture.ts b/packages/perf-harness/src/fixture.ts new file mode 100644 index 0000000000..b7f35935b6 --- /dev/null +++ b/packages/perf-harness/src/fixture.ts @@ -0,0 +1,127 @@ +import { Pool } from 'pg'; + +export const FIXTURE_VERSION = 1; + +export interface PrepareFixtureOptions { + databaseUrl: string; + schema: string; + tables: number; +} + +export interface PreparedFixture { + fixtureVersion: number; + database: string; + serverVersion: string; + schema: string; + tableCount: number; + functionCount: number; +} + +export const validateFixtureSchema = (schema: string): string => { + if ( + !/^cperf_[a-z0-9_]*$/.test(schema) || + schema.length > 63 || + schema.includes('\0') + ) { + throw new Error( + 'fixture schema must start with cperf_, use only lowercase letters, digits, and underscores, and fit PostgreSQL identifiers' + ); + } + return schema; +}; + +export const validateFixtureTableCount = (tables: number): number => { + if (!Number.isSafeInteger(tables) || tables < 1 || tables > 500) { + throw new Error('fixture table count must be an integer between 1 and 500'); + } + return tables; +}; + +const quoteIdentifier = (identifier: string): string => + `"${identifier.replaceAll('"', '""')}"`; + +export const prepareFixture = async ( + options: PrepareFixtureOptions +): Promise => { + const schema = validateFixtureSchema(options.schema); + const tables = validateFixtureTableCount(options.tables); + const quotedSchema = quoteIdentifier(schema); + const pool = new Pool({ connectionString: options.databaseUrl, max: 1 }); + const client = await pool.connect(); + try { + await client.query('begin'); + const existing = await client.query<{ exists: boolean }>( + 'select exists(select 1 from pg_catalog.pg_namespace where nspname = $1) as exists', + [schema] + ); + if (existing.rows[0]?.exists) { + throw new Error( + `fixture schema '${schema}' already exists; this command never replaces schemas` + ); + } + await client.query(`create schema ${quotedSchema}`); + await client.query( + `comment on schema ${quotedSchema} is 'cperf fixture version ${FIXTURE_VERSION}'` + ); + await client.query( + `create type ${quotedSchema}."entity_status" as enum ('draft', 'active', 'archived')` + ); + await client.query(` + create table ${quotedSchema}."account" ( + id bigint generated always as identity primary key, + external_id uuid not null unique, + name text not null, + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() + ) + `); + for (let index = 1; index <= tables; index += 1) { + const table = quoteIdentifier(`entity_${index}`); + const functionName = quoteIdentifier(`entity_${index}_by_account`); + const indexName = quoteIdentifier(`entity_${index}_account_created_idx`); + await client.query(` + create table ${quotedSchema}.${table} ( + id bigint generated always as identity primary key, + account_id bigint not null references ${quotedSchema}."account"(id), + status ${quotedSchema}."entity_status" not null default 'draft', + title text not null, + tags text[] not null default array[]::text[], + metadata jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now(), + unique (account_id, title) + ); + create index ${indexName} + on ${quotedSchema}.${table} (account_id, created_at desc); + create function ${quotedSchema}.${functionName}(requested_account_id bigint) + returns setof ${quotedSchema}.${table} + language sql stable + as 'select * from ${quotedSchema}.${table} where account_id = requested_account_id'; + `); + } + const identity = await client.query<{ + database: string; + server_version: string; + }>( + "select current_database() as database, current_setting('server_version') as server_version" + ); + await client.query('commit'); + return { + fixtureVersion: FIXTURE_VERSION, + database: identity.rows[0].database, + serverVersion: identity.rows[0].server_version, + schema, + tableCount: tables + 1, + functionCount: tables, + }; + } catch (error) { + try { + await client.query('rollback'); + } catch { + // Preserve the fixture preparation error; the client is discarded below. + } + throw error; + } finally { + client.release(); + await pool.end(); + } +}; diff --git a/packages/perf-harness/src/index.ts b/packages/perf-harness/src/index.ts new file mode 100644 index 0000000000..7829d7dc66 --- /dev/null +++ b/packages/perf-harness/src/index.ts @@ -0,0 +1,20 @@ +#!/usr/bin/env node + +export * from './fixture'; +export * from './metrics'; +export * from './process'; +export * from './report'; +export * from './run'; +export * from './schedule'; +export * from './types'; + +import { cliMain } from './run'; + +if (require.main === module) { + void cliMain().catch((error: unknown) => { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n` + ); + process.exitCode = 1; + }); +} diff --git a/packages/perf-harness/src/metrics.ts b/packages/perf-harness/src/metrics.ts new file mode 100644 index 0000000000..007e79fa13 --- /dev/null +++ b/packages/perf-harness/src/metrics.ts @@ -0,0 +1,79 @@ +import { performance } from 'node:perf_hooks'; + +import type { + CaseValidation, + JsonValue, + MemorySnapshot, + SuccessfulWorkerResult, +} from './types'; + +export interface MeasuredCaseResult { + schemaHash: string; + schemaTypeCount: number; + runtimeVerified: true; + caseValidation?: CaseValidation; + metadata?: Record; +} + +const collectGarbage = (): void => { + if (typeof global.gc !== 'function') { + throw new Error('benchmark worker requires Node --expose-gc'); + } + global.gc(); + global.gc(); + global.gc(); +}; + +const memorySnapshot = (): MemorySnapshot => { + const memory = process.memoryUsage(); + return { + rss: memory.rss, + heapTotal: memory.heapTotal, + heapUsed: memory.heapUsed, + external: memory.external, + arrayBuffers: memory.arrayBuffers, + }; +}; + +const memoryDelta = ( + baseline: MemorySnapshot, + afterBuild: MemorySnapshot +): MemorySnapshot => ({ + rss: afterBuild.rss - baseline.rss, + heapTotal: afterBuild.heapTotal - baseline.heapTotal, + heapUsed: afterBuild.heapUsed - baseline.heapUsed, + external: afterBuild.external - baseline.external, + arrayBuffers: afterBuild.arrayBuffers - baseline.arrayBuffers, +}); + +export const measureBenchmarkCase = async ( + caseName: string, + build: () => Promise, + validate: (built: Built) => Promise +): Promise => { + collectGarbage(); + const baseline = memorySnapshot(); + const startedAt = performance.now(); + const built = await build(); + const buildMs = performance.now() - startedAt; + const measured = await validate(built); + collectGarbage(); + const afterBuild = memorySnapshot(); + return { + status: 'ok', + pid: process.pid, + caseName, + buildMs, + schemaHash: measured.schemaHash, + schemaTypeCount: measured.schemaTypeCount, + runtimeVerified: measured.runtimeVerified, + caseValidation: measured.caseValidation ?? { passed: true, errors: [] }, + ...(measured.metadata ? { metadata: measured.metadata } : {}), + memory: { + baseline, + afterBuild, + delta: memoryDelta(baseline, afterBuild), + processPeakRss: process.resourceUsage().maxRSS * 1024, + }, + }; +}; diff --git a/packages/perf-harness/src/process.ts b/packages/perf-harness/src/process.ts new file mode 100644 index 0000000000..dee9097204 --- /dev/null +++ b/packages/perf-harness/src/process.ts @@ -0,0 +1,183 @@ +import { spawn } from 'node:child_process'; + +import type { + BenchmarkCaseDefinition, + WorkerConfigEnvelope, + WorkerResult, +} from './types'; + +export const WORKER_RESULT_PREFIX = 'CPERF_RESULT '; +export const DATABASE_URL_ARGUMENT = 'database-url'; +export const WORKER_CONFIG_ARGUMENT = 'worker-config'; + +export interface ParsedValueArgs { + values: Map; +} + +export interface WorkerProcessArgs { + databaseUrl: string; + envelope: WorkerConfigEnvelope; +} + +export interface SpawnedWorkerResult { + pid: number; + result: WorkerResult; +} + +const lastLines = (value: string, count = 20): string => + value.trim().split('\n').slice(-count).join('\n'); + +export const redactSecret = (value: string, secret: string): string => + secret ? value.replaceAll(secret, '') : value; + +export const parseValueArgs = (args: readonly string[]): ParsedValueArgs => { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if ( + !flag?.startsWith('--') || + value === undefined || + value.startsWith('--') + ) { + throw new Error(`expected --name value near '${flag ?? ''}'`); + } + const name = flag.slice(2); + if (values.has(name)) + throw new Error(`--${name} may only be specified once`); + values.set(name, value); + } + return { values }; +}; + +export const parseWorkerProcessArgs = ( + args: readonly string[] +): WorkerProcessArgs => { + const parsed = parseValueArgs(args); + for (const name of parsed.values.keys()) { + if (name !== DATABASE_URL_ARGUMENT && name !== WORKER_CONFIG_ARGUMENT) { + throw new Error(`unsupported worker argument '--${name}'`); + } + } + const databaseUrl = parsed.values.get(DATABASE_URL_ARGUMENT); + if (!databaseUrl) throw new Error('--database-url is required'); + return { + databaseUrl, + envelope: parseWorkerEnvelope(parsed.values.get(WORKER_CONFIG_ARGUMENT)), + }; +}; + +export const runWorkerProcess = ( + workerPath: string, + databaseUrl: string, + definition: BenchmarkCaseDefinition +): Promise => + new Promise((resolve, reject) => { + const config: WorkerConfigEnvelope = { + caseName: definition.name, + workerConfig: definition.workerConfig, + }; + const child = spawn( + process.execPath, + [ + '--expose-gc', + workerPath, + `--${DATABASE_URL_ARGUMENT}`, + databaseUrl, + `--${WORKER_CONFIG_ARGUMENT}`, + Buffer.from(JSON.stringify(config)).toString('base64url'), + ], + { + env: { + ...process.env, + NODE_ENV: 'production', + GRAPHILE_ENV: 'production', + }, + stdio: ['ignore', 'pipe', 'pipe'], + } + ); + const pid = child.pid; + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { + stdout += chunk; + }); + child.stderr.on('data', (chunk: string) => { + stderr += chunk; + }); + child.once('error', reject); + child.once('close', (code, signal) => { + const resultLine = stdout + .split('\n') + .reverse() + .find((line) => line.startsWith(WORKER_RESULT_PREFIX)); + if (!resultLine) { + reject( + new Error( + redactSecret( + `benchmark worker ${pid ?? 'unknown'} exited without a result ` + + `(code=${String(code)}, signal=${String(signal)})` + + (stderr.trim() ? `\n${lastLines(stderr)}` : ''), + databaseUrl + ) + ) + ); + return; + } + try { + const result = JSON.parse( + resultLine.slice(WORKER_RESULT_PREFIX.length) + ) as WorkerResult; + if (typeof pid !== 'number' || result.pid !== pid) { + throw new Error( + `worker PID mismatch: spawned ${String(pid)}, reported ${String( + result.pid + )}` + ); + } + if (result.caseName !== definition.name) { + throw new Error( + `worker case mismatch: expected ${definition.name}, reported ${result.caseName}` + ); + } + if (result.status === 'ok' && code !== 0) { + throw new Error(`successful worker exited with code ${String(code)}`); + } + resolve({ pid, result }); + } catch (error) { + reject( + new Error( + redactSecret( + `invalid result from benchmark worker ${String(pid)}: ${String( + error instanceof Error ? error.message : error + )}`, + databaseUrl + ) + ) + ); + } + }); + }); + +export const parseWorkerEnvelope = ( + encoded: string | undefined +): WorkerConfigEnvelope => { + if (!encoded) throw new Error('--worker-config is required'); + const parsed = JSON.parse( + Buffer.from(encoded, 'base64url').toString('utf8') + ) as Partial; + if ( + typeof parsed.caseName !== 'string' || + parsed.caseName.length === 0 || + parsed.workerConfig === undefined + ) { + throw new Error('worker configuration envelope is invalid'); + } + return parsed as WorkerConfigEnvelope; +}; + +export const writeWorkerResult = (result: WorkerResult): void => { + process.stdout.write(`${WORKER_RESULT_PREFIX}${JSON.stringify(result)}\n`); +}; diff --git a/packages/perf-harness/src/report.ts b/packages/perf-harness/src/report.ts new file mode 100644 index 0000000000..680852a961 --- /dev/null +++ b/packages/perf-harness/src/report.ts @@ -0,0 +1,123 @@ +import type { + BenchmarkCaseDefinition, + BenchmarkRun, + CaseComparison, + CaseSummary, + MetricComparison, + MetricSummary, + SuccessfulWorkerResult, +} from './types'; + +const median = (values: readonly number[]): number => { + if (values.length === 0) throw new Error('cannot summarize zero values'); + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +}; + +const metricSummary = (values: number[]): MetricSummary => ({ + median: median(values), + min: Math.min(...values), + max: Math.max(...values), + samples: values, +}); + +const successfulResultsFor = ( + runs: readonly BenchmarkRun[], + caseName: string +): SuccessfulWorkerResult[] => + runs + .filter((run) => run.caseName === caseName && run.result.status === 'ok') + .map((run) => run.result as SuccessfulWorkerResult); + +export const summarizeCase = ( + runs: readonly BenchmarkRun[], + caseName: string +): CaseSummary | undefined => { + const results = successfulResultsFor(runs, caseName); + if (results.length === 0) return undefined; + return { + sampleCount: results.length, + buildMs: metricSummary(results.map((result) => result.buildMs)), + heapUsedAfterBuild: metricSummary( + results.map((result) => result.memory.afterBuild.heapUsed) + ), + heapUsedDelta: metricSummary( + results.map((result) => result.memory.delta.heapUsed) + ), + rssAfterBuild: metricSummary( + results.map((result) => result.memory.afterBuild.rss) + ), + rssDelta: metricSummary(results.map((result) => result.memory.delta.rss)), + processPeakRss: metricSummary( + results.map((result) => result.memory.processPeakRss) + ), + }; +}; + +const compareMetric = ( + baseline: MetricSummary, + candidate: MetricSummary +): MetricComparison => ({ + baseline: baseline.median, + candidate: candidate.median, + difference: candidate.median - baseline.median, + percentChange: + baseline.median === 0 + ? null + : ((candidate.median - baseline.median) / Math.abs(baseline.median)) * + 100, +}); + +export const compareCases = ( + baselineCase: string, + candidateCase: string, + baseline: CaseSummary, + candidate: CaseSummary +): CaseComparison => ({ + baselineCase, + candidateCase, + buildMs: compareMetric(baseline.buildMs, candidate.buildMs), + heapUsedAfterBuild: compareMetric( + baseline.heapUsedAfterBuild, + candidate.heapUsedAfterBuild + ), + heapUsedDelta: compareMetric(baseline.heapUsedDelta, candidate.heapUsedDelta), + rssAfterBuild: compareMetric(baseline.rssAfterBuild, candidate.rssAfterBuild), + rssDelta: compareMetric(baseline.rssDelta, candidate.rssDelta), + processPeakRss: compareMetric( + baseline.processPeakRss, + candidate.processPeakRss + ), +}); + +export const validateSchemaGroups = ( + definitions: readonly BenchmarkCaseDefinition[], + runs: readonly BenchmarkRun[] +): { + equivalent: boolean; + hashes: Record; + errors: string[]; +} => { + const groups = new Map>(); + for (const definition of definitions) { + if (!definition.expectedSchemaGroup) continue; + const hashes = + groups.get(definition.expectedSchemaGroup) ?? new Set(); + for (const result of successfulResultsFor(runs, definition.name)) { + hashes.add(result.schemaHash); + } + groups.set(definition.expectedSchemaGroup, hashes); + } + const output: Record = {}; + const errors: string[] = []; + for (const [group, hashes] of groups) { + output[group] = hashes.size === 1 ? [...hashes][0] : null; + if (hashes.size !== 1) { + errors.push(`schema group '${group}' did not produce one schema hash`); + } + } + return { equivalent: errors.length === 0, hashes: output, errors }; +}; diff --git a/packages/perf-harness/src/run.ts b/packages/perf-harness/src/run.ts new file mode 100644 index 0000000000..ea6cbaed89 --- /dev/null +++ b/packages/perf-harness/src/run.ts @@ -0,0 +1,245 @@ +import { mkdir, rename, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; + +import { prepareFixture } from './fixture'; +import { parseValueArgs, redactSecret, runWorkerProcess } from './process'; +import { summarizeCase, validateSchemaGroups } from './report'; +import { makeSchedule, validateCaseDefinitions } from './schedule'; +import type { + BenchmarkCaseDefinition, + BenchmarkReport, + BenchmarkRun, + BenchmarkSuiteDefinition, +} from './types'; + +export interface RunSuiteOptions { + databaseUrl: string; + repetitions: number; + seed: number; + order: string[] | null; + output?: string; +} + +export const runBenchmarkSuite = async ( + suite: BenchmarkSuiteDefinition, + options: RunSuiteOptions, + workerPath: string +): Promise => { + validateCaseDefinitions(suite.cases); + const byName = new Map( + suite.cases.map((definition) => [definition.name, definition]) + ); + const schedule = makeSchedule( + suite.cases, + options.repetitions, + options.seed, + options.order + ); + const runs: BenchmarkRun[] = []; + for (const coordinate of schedule) { + const definition = byName.get(coordinate.caseName)!; + process.stderr.write( + `[${runs.length + 1}/${schedule.length}] repetition ${ + coordinate.repetition + }, ${coordinate.caseName}\n` + ); + try { + const spawned = await runWorkerProcess( + workerPath, + options.databaseUrl, + definition + ); + runs.push({ ...coordinate, result: spawned.result }); + } catch (error) { + runs.push({ + ...coordinate, + result: { + status: 'error', + pid: -1, + caseName: coordinate.caseName, + error: redactSecret( + error instanceof Error ? error.message : String(error), + options.databaseUrl + ), + }, + }); + } + } + const successfulRuns = runs.filter((run) => run.result.status === 'ok'); + const allRunsSucceeded = successfulRuns.length === schedule.length; + const pids = successfulRuns.map((run) => run.result.pid); + const freshProcessPerRun = + allRunsSucceeded && + pids.every((pid) => pid > 0 && pid !== process.pid) && + new Set(pids).size === pids.length; + const caseValidationPassed = + allRunsSucceeded && + successfulRuns.every( + (run) => run.result.status === 'ok' && run.result.caseValidation.passed + ); + const schemaGroups = validateSchemaGroups(suite.cases, runs); + const errors: string[] = []; + for (const run of runs) { + if (run.result.status === 'error') { + errors.push( + `${run.caseName} repetition ${run.repetition}: ${run.result.error}` + ); + } else { + for (const error of run.result.caseValidation.errors) { + errors.push(`${run.caseName} repetition ${run.repetition}: ${error}`); + } + } + } + if (!freshProcessPerRun) { + errors.push('fresh-process validation did not pass for every run'); + } + errors.push(...schemaGroups.errors); + const summaries: Record< + string, + NonNullable> + > = {}; + for (const definition of suite.cases) { + const summary = summarizeCase(runs, definition.name); + if (summary) summaries[definition.name] = summary; + } + return { + format: 'constructive-performance-suite/v1', + generatedAt: new Date().toISOString(), + node: process.version, + platform: process.platform, + architecture: process.arch, + suite, + config: { + repetitions: options.repetitions, + seed: options.seed, + order: options.order, + }, + schedule, + runs, + validation: { + allRunsSucceeded, + freshProcessPerRun, + caseValidationPassed, + schemaGroupsEquivalent: schemaGroups.equivalent, + schemaGroups: schemaGroups.hashes, + errors, + }, + summaries, + }; +}; + +export const writeJsonAtomically = async ( + output: string, + value: unknown +): Promise => { + const absoluteOutput = resolve(output); + await mkdir(dirname(absoluteOutput), { recursive: true }); + const temporary = `${absoluteOutput}.tmp-${process.pid}`; + await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { + encoding: 'utf8', + mode: 0o600, + }); + await rename(temporary, absoluteOutput); + return absoluteOutput; +}; + +const positiveInteger = ( + value: string | undefined, + name: string, + defaultValue: number, + maximum: number +): number => { + if (value === undefined) return defaultValue; + if (!/^\d+$/.test(value)) throw new Error(`--${name} must be an integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > maximum) { + throw new Error(`--${name} must be between 1 and ${maximum}`); + } + return parsed; +}; + +const databaseUrl = (args: ReturnType): string => { + const value = args.values.get('database-url'); + if (!value) throw new Error('--database-url is required'); + return value; +}; + +const stringList = (value: string | undefined): string[] | null => { + if (value === undefined) return null; + const result = value.split(','); + if ( + result.some((item) => item.length === 0 || item.trim() !== item) || + new Set(result).size !== result.length + ) { + throw new Error('list values must be unique exact non-empty strings'); + } + return result; +}; + +const parseCases = (encoded: string): BenchmarkCaseDefinition[] => { + const parsed = JSON.parse(Buffer.from(encoded, 'base64url').toString('utf8')); + if (!Array.isArray(parsed)) + throw new Error('--cases must encode a JSON array'); + return parsed as BenchmarkCaseDefinition[]; +}; + +export const cliMain = async (args = process.argv.slice(2)): Promise => { + const [command, ...rest] = args; + const parsed = parseValueArgs(rest); + if (command === 'prepare') { + const schema = parsed.values.get('schema'); + if (!schema) throw new Error('--schema is required'); + const result = await prepareFixture({ + databaseUrl: databaseUrl(parsed), + schema, + tables: positiveInteger(parsed.values.get('tables'), 'tables', 64, 500), + }); + process.stdout.write(`${JSON.stringify(result)}\n`); + return; + } + if (command !== 'run') { + throw new Error('expected prepare or run command'); + } + const cases = parsed.values.get('cases'); + const worker = parsed.values.get('worker'); + if (!cases || !worker) throw new Error('--cases and --worker are required'); + const report = await runBenchmarkSuite( + { + name: parsed.values.get('suite') ?? 'benchmark-suite', + cases: parseCases(cases), + }, + { + databaseUrl: databaseUrl(parsed), + repetitions: positiveInteger( + parsed.values.get('repetitions'), + 'repetitions', + 3, + 50 + ), + seed: positiveInteger( + parsed.values.get('seed'), + 'seed', + 20260813, + 0xffffffff + ), + order: stringList(parsed.values.get('order')), + output: parsed.values.get('output'), + }, + resolve(worker) + ); + const output = await writeJsonAtomically( + parsed.values.get('output') ?? 'performance-report.json', + report + ); + process.stdout.write( + `${JSON.stringify({ output, validation: report.validation })}\n` + ); + if ( + !report.validation.allRunsSucceeded || + !report.validation.freshProcessPerRun || + !report.validation.caseValidationPassed || + !report.validation.schemaGroupsEquivalent + ) { + process.exitCode = 1; + } +}; diff --git a/packages/perf-harness/src/schedule.ts b/packages/perf-harness/src/schedule.ts new file mode 100644 index 0000000000..8eaa98fb86 --- /dev/null +++ b/packages/perf-harness/src/schedule.ts @@ -0,0 +1,77 @@ +import type { BenchmarkCaseDefinition, BenchmarkCoordinate } from './types'; + +const seededRandom = (seed: number): (() => number) => { + let state = seed >>> 0; + return () => { + state += 0x6d2b79f5; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +}; + +const shuffledCaseNames = ( + definitions: readonly BenchmarkCaseDefinition[], + seed: number, + repetition: number +): string[] => { + const result = definitions.map(({ name }) => name); + const random = seededRandom((seed ^ Math.imul(repetition, 0x9e3779b1)) >>> 0); + for (let index = result.length - 1; index > 0; index -= 1) { + const swapIndex = Math.floor(random() * (index + 1)); + [result[index], result[swapIndex]] = [result[swapIndex], result[index]]; + } + return result; +}; + +export const validateCaseDefinitions = ( + definitions: readonly BenchmarkCaseDefinition[] +): void => { + if (definitions.length === 0) { + throw new Error('benchmark suite must contain at least one case'); + } + const names = definitions.map(({ name }) => name); + if ( + names.some( + (name) => name.length === 0 || name.trim() !== name || name.includes('\0') + ) || + new Set(names).size !== names.length + ) { + throw new Error( + 'benchmark case names must be unique exact non-empty strings' + ); + } +}; + +export const makeSchedule = ( + definitions: readonly BenchmarkCaseDefinition[], + repetitions: number, + seed: number, + exactOrder: readonly string[] | null = null +): BenchmarkCoordinate[] => { + validateCaseDefinitions(definitions); + if (!Number.isSafeInteger(repetitions) || repetitions < 1) { + throw new Error('repetitions must be a positive safe integer'); + } + const expectedNames = definitions.map(({ name }) => name).sort(); + const schedule: BenchmarkCoordinate[] = []; + for (let repetition = 1; repetition <= repetitions; repetition += 1) { + const order = exactOrder + ? [...exactOrder] + : shuffledCaseNames(definitions, seed, repetition); + if ( + order.length !== definitions.length || + new Set(order).size !== definitions.length || + [...order].sort().some((name, index) => name !== expectedNames[index]) + ) { + throw new Error( + 'exact order must contain each benchmark case exactly once' + ); + } + order.forEach((caseName, index) => { + schedule.push({ repetition, position: index + 1, caseName }); + }); + } + return schedule; +}; diff --git a/packages/perf-harness/src/stock-worker.ts b/packages/perf-harness/src/stock-worker.ts new file mode 100644 index 0000000000..c68cc18f0d --- /dev/null +++ b/packages/perf-harness/src/stock-worker.ts @@ -0,0 +1,93 @@ +import { createHash } from 'node:crypto'; + +import { + defaultPreset as graphileBuildPreset, + makeSchema, +} from 'graphile-build'; +import { defaultPreset as graphileBuildPgPreset } from 'graphile-build-pg'; +import { execute, lexicographicSortSchema, parse, printSchema } from 'graphql'; +import { makePgService } from 'postgraphile/adaptors/pg'; + +import { measureBenchmarkCase } from './metrics'; +import { + parseWorkerProcessArgs, + redactSecret, + writeWorkerResult, +} from './process'; + +interface StockConfig { + schemas: string[]; +} + +const validateConfig = (value: unknown): StockConfig => { + const schemas = (value as Partial)?.schemas; + if ( + !Array.isArray(schemas) || + schemas.length === 0 || + schemas.some((schema) => typeof schema !== 'string' || schema.length === 0) + ) { + throw new Error('stock worker requires a non-empty schemas array'); + } + return { schemas }; +}; + +const main = async (): Promise => { + let databaseUrl = ''; + let caseName = 'unknown'; + try { + const workerArgs = parseWorkerProcessArgs(process.argv.slice(2)); + databaseUrl = workerArgs.databaseUrl; + const { envelope } = workerArgs; + caseName = envelope.caseName; + const config = validateConfig(envelope.workerConfig); + const service = makePgService({ + connectionString: databaseUrl, + schemas: config.schemas, + pubsub: false, + }); + try { + const result = await measureBenchmarkCase( + caseName, + async () => + makeSchema({ + extends: [graphileBuildPreset, graphileBuildPgPreset], + pgServices: [service], + }), + async ({ schema }) => { + const execution = await execute({ + schema, + document: parse('{ __typename }'), + }); + if ( + execution.errors?.length || + execution.data?.__typename !== 'Query' + ) { + throw new Error('runtime verification query failed'); + } + const schemaText = printSchema(lexicographicSortSchema(schema)); + return { + schemaHash: createHash('sha256').update(schemaText).digest('hex'), + schemaTypeCount: Object.keys(schema.getTypeMap()).length, + runtimeVerified: true as const, + }; + } + ); + writeWorkerResult(result); + } finally { + await service.release(); + } + } catch (error) { + writeWorkerResult({ + status: 'error', + pid: process.pid, + caseName, + error: redactSecret( + error instanceof Error ? error.message : String(error), + databaseUrl + ), + }); + process.exitCode = 1; + } +}; + +if (require.main === module) void main(); diff --git a/packages/perf-harness/src/types.ts b/packages/perf-harness/src/types.ts new file mode 100644 index 0000000000..3713a4edc4 --- /dev/null +++ b/packages/perf-harness/src/types.ts @@ -0,0 +1,129 @@ +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = + JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + +export interface BenchmarkCaseDefinition { + name: string; + workerConfig: JsonValue; + expectedSchemaGroup?: string; +} + +export interface BenchmarkSuiteDefinition { + name: string; + cases: BenchmarkCaseDefinition[]; +} + +export interface BenchmarkCoordinate { + repetition: number; + position: number; + caseName: string; +} + +export interface WorkerConfigEnvelope { + caseName: string; + workerConfig: JsonValue; +} + +export interface MemorySnapshot { + rss: number; + heapTotal: number; + heapUsed: number; + external: number; + arrayBuffers: number; +} + +export interface CaseValidation { + passed: boolean; + errors: string[]; +} + +export interface SuccessfulWorkerResult { + status: 'ok'; + pid: number; + caseName: string; + buildMs: number; + schemaHash: string; + schemaTypeCount: number; + runtimeVerified: true; + caseValidation: CaseValidation; + metadata?: Record; + memory: { + baseline: MemorySnapshot; + afterBuild: MemorySnapshot; + delta: MemorySnapshot; + processPeakRss: number; + }; +} + +export interface FailedWorkerResult { + status: 'error'; + pid: number; + caseName: string; + error: string; +} + +export type WorkerResult = SuccessfulWorkerResult | FailedWorkerResult; + +export interface BenchmarkRun extends BenchmarkCoordinate { + result: WorkerResult; +} + +export interface MetricSummary { + median: number; + min: number; + max: number; + samples: number[]; +} + +export interface CaseSummary { + sampleCount: number; + buildMs: MetricSummary; + heapUsedAfterBuild: MetricSummary; + heapUsedDelta: MetricSummary; + rssAfterBuild: MetricSummary; + rssDelta: MetricSummary; + processPeakRss: MetricSummary; +} + +export interface MetricComparison { + baseline: number; + candidate: number; + difference: number; + percentChange: number | null; +} + +export interface CaseComparison { + baselineCase: string; + candidateCase: string; + buildMs: MetricComparison; + heapUsedAfterBuild: MetricComparison; + heapUsedDelta: MetricComparison; + rssAfterBuild: MetricComparison; + rssDelta: MetricComparison; + processPeakRss: MetricComparison; +} + +export interface BenchmarkReport { + format: 'constructive-performance-suite/v1'; + generatedAt: string; + node: string; + platform: string; + architecture: string; + suite: BenchmarkSuiteDefinition; + config: { + repetitions: number; + seed: number; + order: string[] | null; + }; + schedule: BenchmarkCoordinate[]; + runs: BenchmarkRun[]; + validation: { + allRunsSucceeded: boolean; + freshProcessPerRun: boolean; + caseValidationPassed: boolean; + schemaGroupsEquivalent: boolean; + schemaGroups: Record; + errors: string[]; + }; + summaries: Record; +} diff --git a/packages/perf-harness/tsconfig.esm.json b/packages/perf-harness/tsconfig.esm.json new file mode 100644 index 0000000000..1a3c9914f1 --- /dev/null +++ b/packages/perf-harness/tsconfig.esm.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "dist/esm", + "module": "esnext", + "moduleResolution": "bundler" + } +} diff --git a/packages/perf-harness/tsconfig.json b/packages/perf-harness/tsconfig.json new file mode 100644 index 0000000000..319daa4b0d --- /dev/null +++ b/packages/perf-harness/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "moduleResolution": "nodenext", + "module": "nodenext", + "isolatedModules": true + }, + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9f681e75ea..ca6ae59d08 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2711,6 +2711,37 @@ importers: version: 0.8.0 publishDirectory: dist + packages/perf-harness: + dependencies: + graphile-build: + specifier: 5.1.1 + version: 5.1.1(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0) + graphile-build-pg: + specifier: 5.1.3 + version: 5.1.3(@dataplan/pg@1.1.1(@dataplan/json@1.0.1(grafast@1.1.2(graphql@16.13.0)))(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.1.2(graphql@16.13.0))(graphile-build@5.1.1(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) + graphile-config: + specifier: 1.1.0 + version: 1.1.0 + graphql: + specifier: 16.13.0 + version: 16.13.0 + pg: + specifier: ^8.21.0 + version: 8.21.0 + postgraphile: + specifier: 5.1.4 + version: 5.1.4(f282a162d8bd20a217e08c60f5396af8) + devDependencies: + '@types/node': + specifier: ^22.19.11 + version: 22.19.19 + '@types/pg': + specifier: ^8.20.4 + version: 8.20.4 + makage: + specifier: ^0.8.0 + version: 0.8.0 + packages/postmaster: dependencies: 12factor-env: