diff --git a/bun.lockb b/bun.lockb index 935deb7a..9a61a74b 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 65fa7826..02d378b4 100644 --- a/package.json +++ b/package.json @@ -49,12 +49,13 @@ "immutable": "^4.3.4", "lru-cache": "^6.0.0", "object-hash": "^2.0.3", - "pgsql-ast-parser": "npm:@tinbase/pgsql-ast-parser@^12.1.0" + "pgsql-ast-parser": "npm:@tinbase/pgsql-ast-parser@^12.2.0" }, "devDependencies": { "@babel/cli": "^7.10.5", "@babel/core": "^7.10.5", "@babel/preset-env": "^7.10.4", + "@electric-sql/pglite": "0.5.4", "@hot-loader/react-dom": "^16.13.0", "@mikro-orm/core": "^5.8.1", "@mikro-orm/postgresql": "^5.8.1", diff --git a/src/column.ts b/src/column.ts index 8facf359..822727fa 100644 --- a/src/column.ts +++ b/src/column.ts @@ -1,7 +1,8 @@ import { _Column, IValue, _IIndex, NotSupported, _Transaction, QueryError, _IType, SchemaField, ChangeHandler, nil, ISubscription, DropHandler } from './interfaces-private'; import type { MemoryTable } from './table'; import { Evaluator } from './evaluator'; -import { ColumnConstraint, AlterColumn, Expr, toSql } from 'pgsql-ast-parser'; +import { ColumnConstraint, AlterColumn, Expr, toSql, DataTypeDef } from 'pgsql-ast-parser'; +import { policiesDependingOn } from './execution/policy-deps'; import { ignore, nullIsh } from './utils'; import { columnEvaluator } from './transforms/selection'; import { BIndex } from './schema/btree-index'; @@ -29,10 +30,18 @@ export class ColRef implements _Column { usedInIndexes = new Set(); private drophandlers = new Set(); + /** + * The type as written in the DDL. pg-mem folds smallint into integer, real into double and + * char(n) into text, but information_schema reports the declared type, and a generator + * reading data_type should see `smallint`, not `integer`. + */ + declaredType: DataTypeDef | nil; + constructor(readonly table: MemoryTable , public expression: Evaluator , _schema: SchemaField , public name: string) { + this.declaredType = (_schema as any)?.declaredType ?? (_schema as any)?.dataType ?? null; } addConstraints(clist: ColumnConstraint[], t: _Transaction): this { @@ -178,6 +187,10 @@ export class ColRef implements _Column { break; case 'set type': const newType = this.table.ownerSchema.getType(alter.dataType); + if (policiesDependingOn(this.table, this.expression.id!).length) { + // postgres stores policies as bound expressions over the column's type + throw new QueryError('cannot alter type of a column used in a policy definition', '0A000'); + } // USING gives an explicit per-row conversion (referencing the // column and any others); without it, an implicit cast is used. const conv = alter.using @@ -235,15 +248,24 @@ export class ColRef implements _Column { } checkConstraints(toInsert: any, t: _Transaction) { + const col = this.expression.get(toInsert, t); + if (this.isSmallint && typeof col === 'number' && (col < -32768 || col > 32767)) { + // smallint is stored as integer here; enforce int2's range like postgres does + throw new QueryError('smallint out of range', '22003'); + } if (!this.notNull) { return; } - const col = this.expression.get(toInsert, t); if (nullIsh(col)) { throw new QueryError(`null value in column "${this.expression.id}" violates not-null constraint`); } } + private get isSmallint() { + const n = (this.declaredType as any)?.name?.toLowerCase(); + return n === 'smallint' || n === 'int2' || n === 'smallserial'; + } + setDefaults(toInsert: any, t: _Transaction) { const col = this.expression.get(toInsert, t); if (col !== undefined) { diff --git a/src/constraints/foreign-key.ts b/src/constraints/foreign-key.ts index 6d70d937..f07c0f78 100644 --- a/src/constraints/foreign-key.ts +++ b/src/constraints/foreign-key.ts @@ -3,6 +3,7 @@ import { Expr, ExprBinary, TableConstraintForeignKey } from 'pgsql-ast-parser'; import { asTable, CreateIndexColDef, _IConstraint, _ITable, _Transaction } from '../interfaces-private'; import { nullIsh } from '../utils'; import { deferCheck } from '../execution/deferred-checks'; +import { enqueueRi } from '../execution/ri-queue'; export class ForeignKey implements _IConstraint { @@ -46,6 +47,11 @@ export class ForeignKey implements _IConstraint { return this.foreignTable.name; } + /** The referenced table, for dependency checks (DROP TABLE / DROP COLUMN refuse to orphan this FK). */ + get referencedTable(): _ITable { + return this.foreignTable; + } + constructor(readonly name: string) { } @@ -124,13 +130,14 @@ export class ForeignKey implements _IConstraint { right: b, }), equals[0]); - // check nothing matches - for (const local of table.selection.filter(expr).enumerate(dt)) { + // check nothing matches - as a queued RI action, see ri-queue.ts + enqueueRi(() => { + for (const local of [...table.selection.filter(expr).enumerate(dt)]) { // ====== ON DELETE switch (neu ? onUpdate : onDelete) { case 'no action': case 'restrict': - throw new QueryError(`update or delete on table "${ftable.name}" violates foreign key constraint on table "${this.name}"`); + throw new QueryError(`update or delete on table "${ftable.name}" violates foreign key constraint "${this.name}" on table "${table.name}"`, '23503'); case 'cascade': if (neu) { for (let i = 0; i < fcols.length; i++) { @@ -150,6 +157,7 @@ export class ForeignKey implements _IConstraint { break; } } + }); })); // ===================== @@ -191,7 +199,7 @@ export class ForeignKey implements _IConstraint { yielded = true; } if (!yielded) { - throw new QueryError(`insert or update on table "${ftable.name}" violates foreign key constraint on table "${this.name}"`); + throw new QueryError(`insert or update on table "${table.name}" violates foreign key constraint "${this.name}"`, '23503'); } }; if (deferred) { @@ -243,8 +251,14 @@ export class ForeignKey implements _IConstraint { return this; } + /** set by the owning table: drops this FK from its constraint list (and so the catalogues) */ + onUninstalled?: () => void; + uninstall(t: _Transaction): void { this.unsubs.forEach(x => x.unsubscribe()); this.unsubs = []; + // also when uninstalled from the other side (DROP TABLE CASCADE), not only + // via the table's own ConstraintWrapper - or the catalogues keep listing a dead FK + this.onUninstalled?.(); } } \ No newline at end of file diff --git a/src/datatypes/datatypes.ts b/src/datatypes/datatypes.ts index bb942743..56467c9a 100644 --- a/src/datatypes/datatypes.ts +++ b/src/datatypes/datatypes.ts @@ -101,6 +101,26 @@ export function isInteger(t: DataType | IType) { return integers.has(type); } +const INT4_MIN = -2147483648; +const INT4_MAX = 2147483647; +const INT8_MIN = BigInt('-9223372036854775808'); +const INT8_MAX = BigInt('9223372036854775807'); + +/** Postgres stores integer as int4: a value outside it is an error, not a silently wider number. */ +export function checkInt4(n: number): number { + if (n < INT4_MIN || n > INT4_MAX) { + throw new QueryError('integer out of range', '22003'); + } + return n; +} + +export function checkInt8(n: bigint): bigint { + if (n < INT8_MIN || n > INT8_MAX) { + throw new QueryError('bigint out of range', '22003'); + } + return n; +} + class NumberType extends TypeBase { constructor(readonly primary: DataType, typeId: number) { @@ -122,12 +142,16 @@ class NumberType extends TypeBase { } doPrefer(type: _IType): _IType | null { + // the wider of the two, as postgres' implicit numeric promotion does: + // int4 < int8 < numeric < float8 (so int = bigint compares as bigint, whichever side it is on) switch (type.primary) { case DataType.integer: - case DataType.bigint: return this; - case DataType.float: + case DataType.bigint: + return this.primary === DataType.integer ? type : this; case DataType.decimal: + return this.primary === DataType.float ? this : type; + case DataType.float: return type; } return null; @@ -157,9 +181,11 @@ class NumberType extends TypeBase { , value , (raw, t) => { const got = value.get(raw, t); - return typeof got === 'number' - ? Math.round(got) - : got; + if (typeof got !== 'number') { + return got; + } + const r = Math.round(got); + return to.primary === DataType.integer ? checkInt4(r) : r; } ); } @@ -244,7 +270,7 @@ class BigIntType extends TypeBase { case DataType.text: return value.setType(to); case DataType.integer: - return value.setType(to).setConversion((s: string) => Number(BigInt(s)), toInt => ({ toInt })); + return value.setType(to).setConversion((s: string) => checkInt4(Number(BigInt(s))), toInt => ({ toInt })); case DataType.float: return value.setType(to).setConversion((s: string) => Number(s), toFloat => ({ toFloat })); case DataType.decimal: @@ -296,7 +322,7 @@ class DecimalType extends TypeBase { case DataType.float: return value.setType(to).setConversion((s: string) => Number(s), toFloat => ({ toFloat })); case DataType.integer: - return value.setType(to).setConversion((s: string) => Number(Decimal.fromText(s).round(0).toString()), toInt => ({ toInt })); + return value.setType(to).setConversion((s: string) => checkInt4(Number(Decimal.fromText(s).round(0).toString())), toInt => ({ toInt })); case DataType.bigint: return value.setType(to).setConversion((s: string) => Decimal.fromText(s).round(0).toString(), toBig => ({ toBig })); } @@ -516,17 +542,46 @@ class TextType extends TypeBase { } if (numbers.has(to.primary)) { - const isInt = integers.has(to.primary); + // parse like postgres' input functions: the whole string (surrounding blanks aside) + // must be a number - parseFloat('12abc') = 12 is not - and produce the representation + // the target type stores (numeric and bigint are digit strings, not JS numbers) + const pgName = to.primary === DataType.decimal ? 'numeric' : to.primary === DataType.float ? 'double precision' : to.primary; + const scale = (to as any).scale as number | null | undefined; return value .setConversion(str => { - const val = Number.parseFloat(str); - if (!Number.isFinite(val)) { - throw new QueryError(`invalid input syntax for ${to.primary}: ${str}`); - } - if (isInt && Math.floor(val) !== val) { - throw new QueryError(`invalid input syntax for ${to.primary}: ${str}`) + const s = String(str).trim(); + const bad = () => new QueryError(`invalid input syntax for type ${pgName}: "${str}"`, '22P02'); + switch (to.primary) { + case DataType.integer: + if (!/^[+-]?\d+$/.test(s)) { + throw bad(); + } + return checkInt4(Number(s)); + case DataType.bigint: + if (!/^[+-]?\d+$/.test(s)) { + throw bad(); + } + return checkInt8(BigInt(s)).toString(); + case DataType.decimal: { + if (!/^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i.test(s) && !/^nan$/i.test(s)) { + throw bad(); + } + const d = Decimal.fromText(s); + return (scale === null || scale === undefined ? d : d.round(scale)).toString(); + } + default: { + if (/^[+-]?inf(inity)?$/i.test(s)) { + return s.startsWith('-') ? -Infinity : Infinity; + } + if (/^nan$/i.test(s)) { + return NaN; + } + if (!/^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i.test(s)) { + throw bad(); + } + return Number(s); + } } - return val; } , castNum => ({ castNum, to: to.primary })); } @@ -906,6 +961,46 @@ export function reconciliateTypes(values: IValue[], nullIfNoMatch?: boolean, str +/** + * Postgres type category (pg_type.typcategory) of the built-in types pg-mem models. + * Postgres has no implicit cast between these categories: a typed value only crosses one + * through an explicit cast, so `uuid_col = text_col` is "operator does not exist: uuid = text". + * Untyped literals and bind parameters are exempt - they are 'unknown' and coerce to anything. + */ +function typeCategory(t: _IType): string | null { + switch (t.primary) { + case DataType.text: + case DataType.citext: + return 'string'; + case DataType.uuid: + return 'uuid'; + case DataType.bool: + return 'bool'; + case DataType.json: + case DataType.jsonb: + return 'json'; + case DataType.integer: + case DataType.bigint: + case DataType.float: + case DataType.decimal: + return 'numeric'; + case DataType.date: + case DataType.timestamp: + case DataType.timestamptz: + return 'datetime'; + case DataType.bytea: + return 'bytea'; + } + return null; +} + +/** True when Postgres refuses to implicitly convert a typed value of `from` into `to`. */ +export function crossesTypeCategory(from: _IType, to: _IType): boolean { + const a = typeCategory(from); + const b = typeCategory(to); + return !!a && !!b && a !== b; +} + /** Finds a common type by implicit conversion */ function reconciliateTypesRaw(values: IValue[], nullIfNoMatch?: false, stricterType?: boolean): _IType; function reconciliateTypesRaw(values: IValue[], nullIfNoMatch: true, stricterType?: boolean): _IType | nil; @@ -926,7 +1021,8 @@ function reconciliateTypesRaw(values: IValue[], nullIfNoMatch?: boolean, stricte // check that all constant literals are matching this. for (const x of values) { - if (!x.isConstantLiteral && !x.type.canConvertImplicit(foundType)) { + const unknownLiteral = x.isConstantLiteral && x.type.primary === DataType.text; + if (!x.isConstantLiteral && !x.type.canConvertImplicit(foundType) || !unknownLiteral && crossesTypeCategory(x.type, foundType)) { if (nullIfNoMatch) { return null; } diff --git a/src/datatypes/t-interval.ts b/src/datatypes/t-interval.ts index 1313ee25..116bde71 100644 --- a/src/datatypes/t-interval.ts +++ b/src/datatypes/t-interval.ts @@ -1,5 +1,5 @@ import { DataType, nil, _IType } from '../interfaces-private'; -import { Interval, normalizeInterval, parseIntervalLiteral } from 'pgsql-ast-parser'; +import { Interval, normalizeInterval, parseIntervalLiteral, intervalToString } from 'pgsql-ast-parser'; import { TypeBase } from './datatype-base'; import { Evaluator } from '../evaluator'; import { intervalToSec } from '../utils'; @@ -31,6 +31,19 @@ export class IntervalType extends TypeBase { return null; } + doCanCast(to: _IType) { + return to.primary === DataType.text; + } + + doCast(value: Evaluator, to: _IType): Evaluator | nil { + if (to.primary !== DataType.text) { + return null; + } + return value + .setType(to) + .setConversion((i: Interval) => pgIntervalText(i), intervalToText => ({ intervalToText })); + } + doEquals(a: Interval, b: Interval): boolean { return intervalToSec(a) === intervalToSec(b); } @@ -41,3 +54,11 @@ export class IntervalType extends TypeBase { return intervalToSec(a) < intervalToSec(b); } } + +/** postgres' default (IntervalStyle = postgres) text form: "1 year 2 mons 3 days 04:05:06", "00:00:00" for zero */ +function pgIntervalText(i: Interval): string { + const txt = String(intervalToString(i)) + .replace(/(-?\d+) months?\b/, (_, n) => `${n} ${Math.abs(Number(n)) === 1 ? 'mon' : 'mons'}`) + .trim(); + return txt || '00:00:00'; +} diff --git a/src/datatypes/t-record.ts b/src/datatypes/t-record.ts index 69b442fa..75f6ca40 100644 --- a/src/datatypes/t-record.ts +++ b/src/datatypes/t-record.ts @@ -59,10 +59,33 @@ export class RecordType extends TypeBase { doCanCast(to: _IType): boolean | nil { // lets say that any type can cast to a record with no columns // this is a hack ... see row_to_json() UT - return to instanceof RecordType && !to.columns.length; + return to instanceof RecordType && !to.columns.length + || to.primary === DataType.text; } doCast(value: Evaluator, to: _IType): Evaluator | nil { + if (to.primary === DataType.text) { + // record_out: (a,b,"c d") - what `'x' || some_row` concatenates in postgres + const cols = this.columns; + return value + .setType(to) + .setConversion((row: any) => '(' + cols.map(c => recordField(row?.[c.name])).join(',') + ')' + , recordToText => ({ recordToText })); + } return value; } } + +/** one field of record_out: empty for NULL, double-quoted when it would otherwise be ambiguous */ +function recordField(v: any): string { + if (v === null || v === undefined) { + return ''; + } + const str = typeof v === 'boolean' ? (v ? 't' : 'f') + : v instanceof Date ? v.toISOString() + : typeof v === 'object' ? JSON.stringify(v) + : String(v); + return str === '' || /[",\\()\s]/.test(str) + ? '"' + str.replace(/(["\\])/g, '$1$1') + '"' + : str; +} diff --git a/src/datatypes/t-timestamp.ts b/src/datatypes/t-timestamp.ts index 4eef6b32..6a6527ec 100644 --- a/src/datatypes/t-timestamp.ts +++ b/src/datatypes/t-timestamp.ts @@ -48,6 +48,17 @@ export class TimestampType extends TypeBase { return null; } + doPrefer(to: _IType): _IType | null { + // date < timestamp < timestamptz: comparing two of them promotes to the wider one, + // as postgres' implicit casts do (date = now() compares as timestamptz) + const rank = (t: DataType) => t === DataType.date ? 1 : t === DataType.timestamp ? 2 : t === DataType.timestamptz ? 3 : 0; + const a = rank(this.primary), b = rank(to.primary); + if (!a || !b) { + return null; + } + return a >= b ? this : to; + } + doCanConvertImplicit(to: _IType) { switch (to.primary) { case DataType.timestamp: diff --git a/src/db.ts b/src/db.ts index 0166b62b..f84dc66e 100644 --- a/src/db.ts +++ b/src/db.ts @@ -9,6 +9,7 @@ import { buildSelection } from './transforms/selection'; import { buildAlias } from './transforms/alias'; import { buildFilter } from './transforms/build-filter'; import { Adapters } from './adapters'; +import { uncacheAll } from './parser/expression-builder'; import { Transaction } from './transaction'; import { buildGroupBy } from './transforms/aggregation'; import { buildLimit } from './transforms/limit'; @@ -81,9 +82,12 @@ class MemoryDb implements _IDb { onSchemaChange() { this.schemaVersion++; + uncacheAll(); this.raiseGlobal('schema-change', this); } + sessionTx: _Transaction | null = null; + constructor(public data: Transaction, schemas?: Map, readonly options: MemoryDbOptions = {}) { if (!schemas) { this.createSchema('public'); diff --git a/src/evaluator.ts b/src/evaluator.ts index 4354ac1c..400d0b30 100644 --- a/src/evaluator.ts +++ b/src/evaluator.ts @@ -411,8 +411,16 @@ export const Value = { if (!Array.isArray(rawArray)) { return false; } - const has = rawArray.some(x => of.equals(rawValue, x)); - return inclusive ? has : !has; + // three-valued logic: NULL IN (…) is NULL, and so is a miss when the list holds a + // NULL - which is why CHECK (col IN ('a','b')) accepts a NULL col in postgres + if (nullIsh(rawValue)) { + return rawArray.length ? null : !inclusive; + } + const has = rawArray.some(x => !nullIsh(x) && of.equals(rawValue, x)); + if (has) { + return inclusive; + } + return rawArray.some(x => nullIsh(x)) ? null : !inclusive; }); }, isNull(leftValue: IValue, expectNull: boolean): IValue { diff --git a/src/execution/plpgsql.ts b/src/execution/plpgsql.ts index d1e2e9ed..6e20af9c 100644 --- a/src/execution/plpgsql.ts +++ b/src/execution/plpgsql.ts @@ -1,4 +1,4 @@ -import { _IDb, _ISchema, _ITable, _Transaction, IValue, _ISelection, QueryError, NotSupported, getId, setId, _IType, Parameter, nil, StatementResult, _IStatementExecutor } from '../interfaces-private'; +import { _IDb, _ISchema, _ITable, _Transaction, IValue, _ISelection, QueryError, NotSupported, getId, setId, _IType, Parameter, nil, StatementResult, _IStatementExecutor, asTable } from '../interfaces-private'; import { Expr, parse, SelectStatement } from 'pgsql-ast-parser'; import { buildValue } from '../parser/expression-builder'; import { buildSelect } from './select'; @@ -42,13 +42,34 @@ function stripComments(code: string): string { ); } -/** Tokenize a plpgsql fragment: strings, numbers, `..` range, `:=`, words, punctuation. */ +/** Tokenize a plpgsql fragment: strings, numbers, `..` range, `:=`, words, operators, punctuation. */ function tokenize(code: string): string[] { // A $tag$…$tag$ (or $$…$$) dollar-quoted block is a single opaque token (matched // first so its content — which may contain ; ' etc. — is not split). JS regex // backreferences let us balance the tag, which the moo lexer can't do. - return stripComments(code) - .match(/\$([a-zA-Z_]\w*)?\$[\s\S]*?\$\1\$|'(?:[^']|'')*'|\d+\.\d+|\d+|\.\.|:=|::|>=|<=|<>|!=|\|\||[a-zA-Z_][\w$]*|[(),.;]|[^\s]/g) ?? []; + const raw = stripComments(code) + .match(/\$([a-zA-Z_]\w*)?\$[\s\S]*?\$\1\$|'(?:[^']|'')*'|\d+\.\d+|\d+|\.\.|:=|::|[+\-*/<>=~!@#%^&|`?]+|[a-zA-Z_][\w$]*|[(),.;]|[^\s]/g) ?? []; + return raw.flatMap(splitOperator); +} + +/** + * Postgres' rule for where an operator token ends: a run of + - * / < > = ~ ! @ # % ^ & | ` ? + * is one operator (->>, #>>, @>, ?|, ~*), except that a multi-character operator cannot end + * in + or - unless it also contains one of ~ ! @ # % ^ & | ` ? - so `x=-1` is `=` then `-1`. + * Splitting `->>` into `- > >` (as listing only two-character operators did) broke every + * trigger that reads JSON, e.g. `new.raw_user_meta_data->>'name'`. + */ +function splitOperator(tok: string): string[] { + if (tok.length < 2 || !/^[+\-*/<>=~!@#%^&|`?]+$/.test(tok) || /[~!@#%^&|`?]/.test(tok)) { + return [tok]; + } + let op = tok; + const tail: string[] = []; + while (op.length > 1 && /[+-]$/.test(op)) { + tail.unshift(op[op.length - 1]); + op = op.slice(0, -1); + } + return [op, ...tail]; } /** strip surrounding single quotes and unescape '' -> ' */ @@ -609,13 +630,15 @@ function makeHelpers(schema: _ISchema, params: Parameter[], returns: _IType | ni /** compiles a regular plpgsql function into a callable implementation */ function buildPlpgsqlFunction(code: string, args: VarDef[], returns: _IType | nil, schema: _ISchema) { - const { decls, block } = new GParser(code).parse(); - const setof = usesSetof([block]); + // parsed now to report syntax errors at CREATE FUNCTION; re-parsed at first call with + // %ROWTYPE / %TYPE resolved, once the tables they name exist + const setof = usesSetof([new GParser(code).parse().block]); // compilation is deferred to first call: the function's own name (recursion) and any // later-defined helpers must already be registered, which they are by call time let program: GProgram | null = null; const compile = (): GProgram => { + const { decls, block } = new GParser(expandRowTypes(tokenize(code), schema)).parse(); const locals = decls.map(d => ({ name: d.name, type: schema.getType(parseTypeDef((d as any).typeSrc)) as _IType, @@ -701,10 +724,12 @@ function mangleTrigger(toks: string[]): string[] { * assignments, control flow, RAISE, EXCEPTION, etc. Compiled lazily per table. */ function buildPlpgsqlTrigger(code: string, schema: _ISchema): TriggerRunner { - const { decls, block } = new GParser(mangleTrigger(tokenize(code))).parse(); + // syntax check at CREATE FUNCTION; compiled per table at first firing (see compileForTable) + new GParser(mangleTrigger(tokenize(code))).parse(); const perTable = new Map<_ITable, { compiled: GCompiled[]; locals: any[]; cols: { id: string }[] }>(); const compileForTable = (table: _ITable) => { + const { decls, block } = new GParser(expandRowTypes(mangleTrigger(tokenize(code)), schema)).parse(); const cols = table.selection.columns .filter(c => !!c.id) .map(c => ({ id: c.id!, type: c.type })); @@ -777,6 +802,68 @@ function buildPlpgsqlTrigger(code: string, schema: _ISchema): TriggerRunner { }; } +/** a table named by a plpgsql declaration (`markets` / `public.markets`) */ +function declTable(schema: _ISchema, ref: string[]): _ITable { + const name = ref.length > 1 ? { schema: ref[0], name: ref[1] } : { name: ref[0] }; + return asTable(schema.getThisOrSiblingFor(name).getObject(name)); +} + +/** + * Row-typed variables, by expansion into scalars (as NEW/OLD are, see mangleTrigger): + * DECLARE m public.markets%ROWTYPE; -> one variable per column, __r_m__ + * m.col -> __r_m__col + * SELECT … INTO m -> SELECT … INTO __r_m__c1, __r_m__c2, … (positional, like postgres) + * and DECLARE x tbl.col%TYPE takes that column's type. + */ +function expandRowTypes(toks: string[], schema: _ISchema): string[] { + const lower = toks.map(t => t.toLowerCase()); + const di = lower.indexOf('declare'); + const bi = lower.indexOf('begin'); + if (di < 0 || bi < di) { + return toks; + } + const rowVars = new Map(); + const out: string[] = toks.slice(0, di + 1); + for (let i = di + 1; i < bi;) { + let j = i; + while (j < bi && toks[j] !== ';') { + j++; + } + const decl = toks.slice(i, j); + const pct = decl.indexOf('%'); + const kind = pct > 1 ? decl[pct + 1]?.toLowerCase() : null; + // the name, possibly qualified, before %: [a, '.', b] -> ['a', 'b'] + const ref = pct > 1 ? decl.slice(1, pct).filter(x => x !== '.') : []; + if (kind === 'rowtype') { + const name = decl[0].toLowerCase(); + const cols = declTable(schema, ref).selection.columns.filter(c => !!c.id); + for (const c of cols) { + out.push(`__r_${name}__${c.id}`, c.type.name, ';'); + } + rowVars.set(name, cols.map(c => `__r_${name}__${c.id}`)); + } else if (kind === 'type' && ref.length >= 2) { + const col = ref[ref.length - 1]; + const type = declTable(schema, ref.slice(0, -1)).getColumnRef(col).expression.type; + out.push(decl[0], type.name, ...decl.slice(pct + 2), ';'); + } else { + out.push(...decl, ';'); + } + i = j + 1; + } + for (let k = bi; k < toks.length; k++) { + const vars = rowVars.get(lower[k]); + if (vars && toks[k + 1] === '.' && /^[a-zA-Z_]/.test(toks[k + 2] ?? '')) { + out.push(`__r_${lower[k]}__${lower[k + 2]}`); + k += 2; + } else if (vars && (lower[k - 1] === 'into' && lower[k - 2] !== 'insert' || lower[k - 1] === 'strict' && lower[k - 2] === 'into')) { + vars.forEach((v, n) => out.push(...(n ? [','] : []), v)); + } else { + out.push(toks[k]); + } + } + return out; +} + function parseTypeDef(typeSrc: string): any { const e = parseExpr(`null::${typeSrc}`); if (e.type !== 'cast') { diff --git a/src/execution/policy-deps.ts b/src/execution/policy-deps.ts new file mode 100644 index 00000000..f04b9feb --- /dev/null +++ b/src/execution/policy-deps.ts @@ -0,0 +1,99 @@ +import { astVisitor, Expr, SelectFromStatement } from 'pgsql-ast-parser'; +import { _ITable, nil } from '../interfaces-private'; + +/** A column a policy predicate reads. */ +export interface ColumnUse { + table: _ITable; + column: string; +} + +/** + * The (table, column) pairs a policy's USING / WITH CHECK reads, with SQL scoping: a ref + * qualified by an alias or table name resolves through the FROM clauses in scope, innermost + * first; an unqualified one resolves to the innermost table that has the column, the policy's + * own table being the outermost scope. So in a policy on `scenes`, + * `exists (select 1 from storyboards s where s.id = storyboard_id)` reads storyboards.id and + * scenes.storyboard_id - and not scenes.id. + * + * Postgres records these as dependencies: the columns cannot be dropped or retyped while the + * policy exists. + */ +export function policyColumnUses(owner: _ITable, ...exprs: (Expr | nil)[]): ColumnUse[] { + const schema = owner.ownerSchema; + const scopes: Map[] = [new Map([[owner.name, owner]])]; + const uses: ColumnUse[] = []; + + const hasColumn = (t: _ITable, c: string) => { + try { + return !!t.getColumnRef?.(c, true); + } catch { + return false; + } + }; + + const visitor = astVisitor(v => ({ + selection: (s: SelectFromStatement) => { + const scope = new Map(); + for (const f of s.from ?? []) { + if (f.type !== 'table') { + continue; + } + const t = schema.getObject(f.name, { nullIfNotFound: true }) as _ITable | nil; + if (t && (t as any).type === 'table') { + scope.set(f.name.alias ?? f.name.name, t); + } + } + scopes.push(scope); + try { + v.super().selection(s); + } finally { + scopes.pop(); + } + return s; + }, + ref: r => { + if (r.name === '*') { + return r; + } + for (let i = scopes.length - 1; i >= 0; i--) { + if (r.table) { + const t = scopes[i].get(r.table.name); + if (t) { + uses.push({ table: t, column: r.name }); + return r; + } + continue; + } + for (const t of scopes[i].values()) { + if (hasColumn(t, r.name)) { + uses.push({ table: t, column: r.name }); + return r; + } + } + } + return r; + }, + })); + + for (const e of exprs) { + if (e) { + visitor.expr(e); + } + } + return uses; +} + +/** Policies, on any table of the db, that read `table.column`. */ +export function policiesDependingOn(table: _ITable, column: string): { table: _ITable; policy: string }[] { + const out: { table: _ITable; policy: string }[] = []; + for (const schema of table.ownerSchema.db.listSchemas()) { + for (const t of schema.listTables()) { + for (const p of t.rls?.policies ?? []) { + if (policyColumnUses(t, p.using, p.withCheck).some(u => u.table === table && u.column === column)) { + out.push({ table: t, policy: p.name }); + } + } + } + } + return out; +} diff --git a/src/execution/records-mutations/deletion.ts b/src/execution/records-mutations/deletion.ts index fe6c1261..45c05a6f 100644 --- a/src/execution/records-mutations/deletion.ts +++ b/src/execution/records-mutations/deletion.ts @@ -1,5 +1,5 @@ import { _ITable, _Transaction, IValue, _Explainer, _ISchema, asTable, _ISelection, _IIndex, _IStatement } from '../../interfaces-private'; -import { applyReadRls } from '../rls-enforce'; +import { applyReadRls, statementReadsColumns } from '../rls-enforce'; import { fireRowTriggers, fireStatementTriggers, SKIP_ROW } from '../triggers'; import { DeleteStatement } from 'pgsql-ast-parser'; import { MutationDataSourceBase } from './mutation-base'; @@ -12,7 +12,7 @@ export class Deletion extends MutationDataSourceBase { const { schema } = buildCtx(); const table = asTable(schema.getObject(ast.from)); // row-level security: DELETE can only affect rows visible via DELETE policies - const mutatedSel = applyReadRls(table, table.selection, 'delete') + const mutatedSel = applyReadRls(table, table.selection, 'delete', statementReadsColumns(ast as any)) .filter(ast.where); super(table, mutatedSel, ast); diff --git a/src/execution/records-mutations/insert.ts b/src/execution/records-mutations/insert.ts index 6873c91d..7bc96115 100644 --- a/src/execution/records-mutations/insert.ts +++ b/src/execution/records-mutations/insert.ts @@ -1,5 +1,5 @@ import { _ITable, _Transaction, IValue, _Explainer, nil, _ISchema, asTable, _ISelection, _IIndex, QueryError, OnConflictHandler, ChangeOpts, _IStatement, NotSupported } from '../../interfaces-private'; -import { checkWriteRls } from '../rls-enforce'; +import { checkWriteRls, checkConflictUpdateRls } from '../rls-enforce'; import { fireRowTriggers, fireStatementTriggers, SKIP_ROW } from '../triggers'; import { InsertStatement } from 'pgsql-ast-parser'; import { buildValue } from '../../parser/expression-builder'; @@ -117,6 +117,9 @@ export class Insert extends MutationDataSourceBase { ignoreConflicts = { onIndex, update: (item, excluded, t) => { + // row-level security: the conflicting row must be one this role may update + checkConflictUpdateRls(this.table, item, t); + // build setter context const jitem = subject.buildItem(item, excluded); @@ -130,6 +133,9 @@ export class Insert extends MutationDataSourceBase { // execute set setter(t, item, jitem); + + // ... and the updated row must satisfy UPDATE WITH CHECK + checkWriteRls(this.table, 'update', item, t); }, } } diff --git a/src/execution/records-mutations/update.ts b/src/execution/records-mutations/update.ts index 7e2ef026..6394a601 100644 --- a/src/execution/records-mutations/update.ts +++ b/src/execution/records-mutations/update.ts @@ -1,5 +1,5 @@ import { _ITable, _Transaction, _Explainer, _ISchema, asTable, _ISelection, _IIndex, _IStatement } from '../../interfaces-private'; -import { applyReadRls, checkWriteRls } from '../rls-enforce'; +import { applyReadRls, checkWriteRls, statementReadsColumns } from '../rls-enforce'; import { fireRowTriggers, fireStatementTriggers, SKIP_ROW } from '../triggers'; import { UpdateStatement } from 'pgsql-ast-parser'; import { MutationDataSourceBase, Setter, createSetter } from './mutation-base'; @@ -63,7 +63,7 @@ export class Update extends MutationDataSourceBase { // => REGULAR UPDATE // row-level security: UPDATE can only affect rows visible via UPDATE policies - mutatedSel = applyReadRls(into, into.selection, 'update') + mutatedSel = applyReadRls(into, into.selection, 'update', statementReadsColumns(ast as any)) .filter(ast.where); } diff --git a/src/execution/ri-queue.ts b/src/execution/ri-queue.ts new file mode 100644 index 00000000..fb679dde --- /dev/null +++ b/src/execution/ri-queue.ts @@ -0,0 +1,34 @@ +/** + * Referential actions run the way postgres runs its RI triggers: queued as AFTER events and fired + * in order once the statement's own rows are done. A cascade's own RI events are appended to the + * SAME queue (postgres runs RI actions with fire_triggers = false), so actions run breadth-first: + * deleting a user checks a direct NO ACTION reference BEFORE a grandchild cascade (user -> + * profile -> rows) has removed the rows - and fails, like postgres does. + * + * Each statement execution has its own queue (a statement run by a trigger fires its RI events at + * its own end). Outside any statement (the JS table API) actions run immediately. + */ +const queues: (() => void)[][] = []; + +export function withRiQueue(run: () => T): T { + const queue: (() => void)[] = []; + queues.push(queue); + try { + const ret = run(); + while (queue.length) { + queue.shift()!(); + } + return ret; + } finally { + queues.pop(); + } +} + +export function enqueueRi(action: () => void): void { + const queue = queues[queues.length - 1]; + if (queue) { + queue.push(action); + } else { + action(); + } +} diff --git a/src/execution/rls-enforce.ts b/src/execution/rls-enforce.ts index 55e33c10..b21cbf7e 100644 --- a/src/execution/rls-enforce.ts +++ b/src/execution/rls-enforce.ts @@ -5,9 +5,104 @@ import { buildValue } from '../parser/expression-builder'; import { withSelection } from '../parser/context'; import { currentRole } from './roles'; import { Policy, policyAppliesToCommand, policyAppliesToRole } from './rls'; +import { astVisitor, Expr, FromTable } from 'pgsql-ast-parser'; export type RlsCommand = 'select' | 'insert' | 'update' | 'delete'; +/** tables read in a predicate's subqueries (FROM / JOIN), resolved against the table's schema */ +function tablesReadBy(owner: _ITable, exprs: (Expr | null | undefined)[]): _ITable[] { + const out = new Set<_ITable>(); + const v = astVisitor(() => ({ + fromTable: (f: FromTable) => { + const obj = owner.ownerSchema.getThisOrSiblingFor(f.name).getObject(f.name, { nullIfNotFound: true }); + if (obj && (obj as any).type === 'table') { + out.add(obj as _ITable); + } + return f; + }, + })); + for (const e of exprs) { + if (e) { + v.expr(e); + } + } + return [...out]; +} + +/** does any of these predicates contain a subquery ("sublink")? */ +function hasSubLinks(exprs: (Expr | null | undefined)[]): boolean { + let found = false; + const v = astVisitor(() => ({ selection: () => { found = true; return null as any; } })); + for (const e of exprs) { + if (e && !found) { + v.expr(e); + } + } + return found; +} + +/** + * Postgres expands the policies of the relations a query touches - and, through the subqueries in + * those policies, of the relations they read - and refuses a cycle: a profiles policy that queries + * profiles (the usual "admins can see everyone" attempt) is "infinite recursion detected in policy + * for relation profiles", for any role RLS applies to and whether or not there are rows. + * + * Like its fireRIRrules, a relation joins the set being expanded, and is checked against it, only + * when its applicable policies contain a subquery: an UPDATE policy reading its own table whose + * SELECT policy is just `true` is fine. Returns the re-entered relation. + */ +function recursivePolicyRelation(table: _ITable, commands: RlsCommand[], roleName: string, expanding = new Set<_ITable>()): _ITable | null { + if (!table.rls.enabled) { + return null; + } + const exprs = table.rls.policies + .filter(p => policyAppliesToRole(p, roleName) && commands.some(c => policyAppliesToCommand(p, c))) + .flatMap(p => [p.using, p.withCheck]); + if (!hasSubLinks(exprs)) { + return null; + } + if (expanding.has(table)) { + return table; + } + expanding.add(table); + try { + for (const read of tablesReadBy(table, exprs)) { + // a subquery reads: its table's SELECT policies get expanded + const hit = recursivePolicyRelation(read, ['select'], roleName, expanding); + if (hit) { + return hit; + } + } + return null; + } finally { + expanding.delete(table); + } +} + +const recursionCache = new WeakMap<_ITable, Map>(); + +/** Throws postgres' error when expanding this table's policies for the current role recurses. */ +export function assertNoPolicyRecursion(table: _ITable, command: RlsCommand, t: _Transaction, readsColumns = false): void { + const role = currentRole(t).name; + // an UPDATE/DELETE that reads columns (WHERE, RETURNING, SET x = ) needs SELECT rights, + // so the table's SELECT policies are expanded too + const commands: RlsCommand[] = readsColumns && command !== 'select' ? [command, 'select'] : [command]; + // policies can change (DDL): key the cache on them too + const key = `${role}|${commands.join(',')}|${(table.ownerSchema.db as any).schemaVersion}`; + let byKey = recursionCache.get(table); + if (!byKey) { + recursionCache.set(table, byKey = new Map()); + } + let hit = byKey.get(key); + if (hit === undefined) { + hit = recursivePolicyRelation(table, commands, role); + byKey.set(key, hit); + } + if (hit) { + throw new QueryError(`infinite recursion detected in policy for relation "${hit.name}"`, '42P17'); + } +} + /** True when the current role skips RLS entirely (superuser or BYPASSRLS). */ export function bypassesRls(t: _Transaction): boolean { const role = currentRole(t); @@ -56,17 +151,39 @@ function rowPasses(compiled: CompiledPolicy[], kind: 'using' | 'withCheck', role return sawPermissive && permissiveOk && restrictiveOk; } +/** tables whose policies are being compiled: a policy subquery reading one of them is a policy recursion */ +const compiling = new Set<_ITable>(); + /** Runtime read-visibility filter: applied only when RLS is on and the role doesn't bypass. */ class RlsSelection extends FilterBase { - private compiled: CompiledPolicy[]; + /** null: built while compiling this table's own policies (recursion, reported at run time) */ + private compiled: CompiledPolicy[] | null; get index() { return null; } - constructor(private sel: _ISelection, private table: _ITable, private command: RlsCommand) { + constructor(private sel: _ISelection, private table: _ITable, private command: RlsCommand, private readsColumns = false) { super(sel); - this.compiled = compilePolicies(sel, table.rls.policies); + if (compiling.has(table)) { + // compiling would recurse forever. Postgres reports this only for the roles whose + // policies actually recurse, so leave it to assertNoPolicyRecursion at run time. + this.compiled = null; + return; + } + compiling.add(table); + try { + this.compiled = compilePolicies(sel, table.rls.policies); + } finally { + compiling.delete(table); + } + } + + private policies(): CompiledPolicy[] { + if (!this.compiled) { + throw new QueryError(`infinite recursion detected in policy for relation "${this.table.name}"`, '42P17'); + } + return this.compiled; } entropy(t: _Transaction) { @@ -81,8 +198,23 @@ class RlsSelection extends FilterBase { if (!this.enforced(t)) { return this.sel.hasItem(raw, t); } + // index lookups (WHERE id = …) check rows here rather than enumerating + assertNoPolicyRecursion(this.table, this.command, t, this.readsColumns); return this.sel.hasItem(raw, t) - && rowPasses(this.compiled, 'using', currentRole(t).name, this.command, raw, t); + && rowPasses(this.policies(), 'using', currentRole(t).name, this.command, raw, t); + } + + /** + * WHERE on a policed table. Index-based filters (id = 1, IN, ranges) take their index from the + * column's origin - the raw table - and become a selection over it, which silently dropped this + * layer: under RLS, `select * from t where id = 2` returned rows no policy allowed. Filter the + * unprotected selection (keeping index lookups) and re-apply the policies on top. + */ + filter(where: Expr | undefined | null): _ISelection { + if (!where) { + return this; + } + return new RlsSelection(this.sel.filter(where), this.table, this.command, this.readsColumns); } private enforced(t: _Transaction): boolean { @@ -94,9 +226,11 @@ class RlsSelection extends FilterBase { yield* this.sel.enumerate(t); return; } + assertNoPolicyRecursion(this.table, this.command, t, this.readsColumns); const roleName = currentRole(t).name; + const compiled = this.policies(); for (const raw of this.sel.enumerate(t)) { - if (rowPasses(this.compiled, 'using', roleName, this.command, raw, t)) { + if (rowPasses(compiled, 'using', roleName, this.command, raw, t)) { yield raw; } } @@ -112,11 +246,24 @@ class RlsSelection extends FilterBase { } /** Wrap a table's selection with row-level security read enforcement. */ -export function applyReadRls(table: _ITable, selection: _ISelection, command: RlsCommand): _ISelection { +export function applyReadRls(table: _ITable, selection: _ISelection, command: RlsCommand, readsColumns = false): _ISelection { if (!table.rls.policies.length && !table.rls.enabled) { return selection; } - return new RlsSelection(selection, table, command); + return new RlsSelection(selection, table, command, readsColumns); +} + +/** does an UPDATE/DELETE read the table's columns (and so need SELECT rights)? */ +export function statementReadsColumns(ast: { where?: Expr | null; returning?: any; sets?: { value: Expr }[] }): boolean { + if (ast.where || ast.returning?.length) { + return true; + } + let found = false; + const v = astVisitor(() => ({ ref: r => { found = true; return r; } })); + for (const set of ast.sets ?? []) { + v.expr(set.value); + } + return found; } /** Throws if a written row violates the WITH CHECK predicates for a command. */ @@ -124,8 +271,25 @@ export function checkWriteRls(table: _ITable, command: 'insert' | 'update', row: if (!table.rls.enabled || bypassesRls(t)) { return; } + assertNoPolicyRecursion(table, command, t); const compiled = compilePolicies(table.selection, table.rls.policies); if (!rowPasses(compiled, 'withCheck', currentRole(t).name, command, row, t)) { throw new QueryError(`new row violates row-level security policy for table "${table.name}"`, '42501'); } } + +/** + * ON CONFLICT DO UPDATE on an RLS table: the existing row must pass the UPDATE (and, since the + * statement reads it, SELECT) USING policies. Postgres errors rather than skipping the row. + */ +export function checkConflictUpdateRls(table: _ITable, existing: Row, t: _Transaction): void { + if (!table.rls.enabled || bypassesRls(t)) { + return; + } + assertNoPolicyRecursion(table, 'update', t, true); + const compiled = compilePolicies(table.selection, table.rls.policies); + const role = currentRole(t).name; + if (!rowPasses(compiled, 'using', role, 'update', existing, t) || !rowPasses(compiled, 'using', role, 'select', existing, t)) { + throw new QueryError(`new row violates row-level security policy (USING expression) for table "${table.name}"`, '42501'); + } +} diff --git a/src/execution/roles.ts b/src/execution/roles.ts index f5da8b07..2567039c 100644 --- a/src/execution/roles.ts +++ b/src/execution/roles.ts @@ -65,6 +65,33 @@ export function currentRole(t: _Transaction): Role { return getRole(t, currentRoleName(t)) ?? DEFAULT_ROLE; } +/** GRANT … TO / CREATE POLICY … TO a role that does not exist fails in postgres ("role … does not exist"). */ +export function assertRolesExist(t: _Transaction, names: (string | undefined)[]): void { + for (const n of names) { + if (!n || n === 'public' || n === 'current_user' || n === 'session_user' || n === 'current_role') { + continue; + } + if (!getRole(t, n)) { + throw new QueryError(`role "${n}" does not exist`, '42704'); + } + } +} + +/** + * Run `fn` as `role` - a SECURITY DEFINER function's body runs with its owner's rights (so a + * helper like is_member(crew_id) can read a table whose policies would otherwise filter it, + * the standard way to avoid recursive RLS) - then restore the caller's role. + */ +export function runAsRole(t: _Transaction, role: string, fn: () => T): T { + const prev = t.getMap(GLOBAL_VARS).get(CURRENT_ROLE_KEY) as string | undefined; + setGuc(t, CURRENT_ROLE_KEY, role); + try { + return fn(); + } finally { + setGuc(t, CURRENT_ROLE_KEY, prev); + } +} + function setGuc(t: _Transaction, key: string, value: string | undefined): void { let g = t.getMap(GLOBAL_VARS); g = value === undefined ? g.delete(key) : g.set(key, value); diff --git a/src/execution/schema-amends/alter-enum.ts b/src/execution/schema-amends/alter-enum.ts index 5bbe7449..49386e88 100644 --- a/src/execution/schema-amends/alter-enum.ts +++ b/src/execution/schema-amends/alter-enum.ts @@ -17,9 +17,6 @@ export class AlterEnum extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because the index sequence creation does support further rollbacks) - t = t.fullCommit(); const enumValues = this.originalEnum.values switch (this.p.change.type) { @@ -32,9 +29,6 @@ export class AlterEnum extends ExecHelper implements _IStatementExecutor { break; } - // new implicit transaction - t = t.fork(); - return this.noData(t, 'ALTER'); } } diff --git a/src/execution/schema-amends/alter-index.ts b/src/execution/schema-amends/alter-index.ts index 64233308..fbdceb57 100644 --- a/src/execution/schema-amends/alter-index.ts +++ b/src/execution/schema-amends/alter-index.ts @@ -13,7 +13,6 @@ export class AlterIndex extends ExecHelper implements _IStatementExecutor { execute(t: _Transaction) { // commit pending data before making schema changes (no rollback support) - t = t.fullCommit(); const idx = asIndex(this.schema.getObject(this.p.index, { nullIfNotFound: this.p.ifExists, @@ -34,8 +33,6 @@ export class AlterIndex extends ExecHelper implements _IStatementExecutor { } else { ignore(this.p); } - - t = t.fork(); return this.noData(t, 'ALTER INDEX'); } } diff --git a/src/execution/schema-amends/alter-sequence.ts b/src/execution/schema-amends/alter-sequence.ts index a995af8d..8210ee18 100644 --- a/src/execution/schema-amends/alter-sequence.ts +++ b/src/execution/schema-amends/alter-sequence.ts @@ -19,16 +19,10 @@ export class AlterSequence extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because the index sequence creation does support further rollbacks) - t = t.fullCommit(); // alter the sequence this.seq?.alter(t, this.p.change); - // new implicit transaction - t = t.fork(); - return this.noData(t, 'ALTER'); } } diff --git a/src/execution/schema-amends/alter.ts b/src/execution/schema-amends/alter.ts index a04796e5..0a078401 100644 --- a/src/execution/schema-amends/alter.ts +++ b/src/execution/schema-amends/alter.ts @@ -2,6 +2,7 @@ import { _ISchema, _Transaction, SchemaField, NotSupported, _ITable, _IStatement import { AlterTableStatement } from 'pgsql-ast-parser'; import { ignore } from '../../utils'; import { ExecHelper } from '../exec-utils'; +import { policiesDependingOn } from '../policy-deps'; export class Alter extends ExecHelper implements _IStatementExecutor { @@ -17,10 +18,6 @@ export class Alter extends ExecHelper implements _IStatementExecutor { let ignored = 0; - // commit pending data before making changes - // (because does not support further rollbacks) - t = t.fullCommit(); - for (const change of this.p.changes) { function ignoreChange() { @@ -51,6 +48,29 @@ export class Alter extends ExecHelper implements _IStatementExecutor { if (!col) { ignoreChange(); } else { + // a foreign key into this column, or a policy reading it, depends on it: + // postgres refuses unless CASCADE, which drops those dependents too + const name = change.column.name; + const fks = this.table.referencingForeignKeys?.([name]) ?? []; + const policies = policiesDependingOn(this.table, name); + if (change.behaviour === 'cascade') { + for (const fk of fks) { + fk.uninstall(t); + } + for (const p of policies) { + p.table.dropPolicy(p.policy, true); + } + } else if (fks.length || policies.length) { + throw new QueryError({ + error: `cannot drop column ${name} of table ${this.table.name} because other objects depend on it`, + details: [ + ...fks.map(fk => `constraint ${fk.name} on table ${fk.tableName} depends on column ${name} of table ${this.table.name}`), + ...policies.map(p => `policy ${p.policy} on table ${p.table.name} depends on column ${name} of table ${this.table.name}`), + ].join('\n'), + hint: 'Use DROP ... CASCADE to drop the dependent objects too.', + code: '2BP01', + }); + } col.drop(t); } break; @@ -91,10 +111,6 @@ export class Alter extends ExecHelper implements _IStatementExecutor { } } - - - // new implicit transaction - t = t.fork(); return this.noData(t, 'ALTER'); } } diff --git a/src/execution/schema-amends/comment.ts b/src/execution/schema-amends/comment.ts index 8d799290..6d039798 100644 --- a/src/execution/schema-amends/comment.ts +++ b/src/execution/schema-amends/comment.ts @@ -13,9 +13,6 @@ export class Comment extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because does not support further rollbacks) - t = t.fullCommit(); const on = this.p.on; switch (on.type) { diff --git a/src/execution/schema-amends/create-composite-type.ts b/src/execution/schema-amends/create-composite-type.ts index 0e8ac20e..4ea46d3f 100644 --- a/src/execution/schema-amends/create-composite-type.ts +++ b/src/execution/schema-amends/create-composite-type.ts @@ -20,9 +20,7 @@ export class CreateCompositeType extends ExecHelper implements _IStatementExecut } execute(t: _Transaction): StatementResult { - t = t.fullCommit(); new CompositeType(this.onSchema, this.name, this.columns).install(); - t = t.fork(); return this.noData(t, 'CREATE TYPE'); } } diff --git a/src/execution/schema-amends/create-domain.ts b/src/execution/schema-amends/create-domain.ts index 87389e49..923d9844 100644 --- a/src/execution/schema-amends/create-domain.ts +++ b/src/execution/schema-amends/create-domain.ts @@ -60,9 +60,7 @@ export class CreateDomain extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction): StatementResult { - t = t.fullCommit(); new DomainType(this.onSchema, this.name, this.base, this.notNull, this.checks).install(); - t = t.fork(); return this.noData(t, 'CREATE DOMAIN'); } } diff --git a/src/execution/schema-amends/create-enum.ts b/src/execution/schema-amends/create-enum.ts index 7ba24169..6ebc3be4 100644 --- a/src/execution/schema-amends/create-enum.ts +++ b/src/execution/schema-amends/create-enum.ts @@ -15,16 +15,10 @@ export class CreateEnum extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because does not support further rollbacks) - t = t.fullCommit(); // register enum this.onSchema .registerEnum(this.name, this.values); - - // new implicit transaction - t = t.fork(); return this.noData(t, 'CREATE'); } } diff --git a/src/execution/schema-amends/create-function.ts b/src/execution/schema-amends/create-function.ts index bdb5de9b..b66a4a3c 100644 --- a/src/execution/schema-amends/create-function.ts +++ b/src/execution/schema-amends/create-function.ts @@ -3,13 +3,16 @@ import { CreateFunctionStatement } from 'pgsql-ast-parser'; import { ExecHelper } from '../exec-utils'; import { buildValue } from '../../parser/expression-builder'; import { Types } from '../../datatypes'; -import { ignore, deepEqual } from '../../utils'; +import { ignore, deepEqual, executionCtx } from '../../utils'; +import { runAsRole, currentRoleName, DEFAULT_ROLE_NAME } from '../roles'; +import { getTriggerRunner } from '../plpgsql'; import { withSelection } from '../../parser/context'; export class CreateFunction extends ExecHelper implements _IStatementExecutor { private onSchema: _ISchema; private toRegister: FunctionDefinition; private replace: boolean; + private owner: string = DEFAULT_ROLE_NAME; constructor({ schema }: _IStatement, fn: CreateFunctionStatement) { super(fn); @@ -20,9 +23,12 @@ export class CreateFunction extends ExecHelper implements _IStatementExecutor { const lang = schema.db.getLanguage(fn.language.name); - // SECURITY DEFINER/INVOKER is a privilege concept; pg-mem has no roles, so - // the body simply runs with the caller's (only) rights either way. + // SECURITY DEFINER: the body runs as the function's owner (the role that created it), + // see the wrapping below; INVOKER (the default) runs as the caller ignore(fn.security); + // SET search_path = ... / COST / ROWS / PARALLEL: accepted, no effect here (pg-mem resolves + // names without a per-function search_path, and has no planner costs or parallelism) + ignore(fn.settings, fn.cost, fn.rows, fn.parallel); // determine arg types const args = withSelection(schema.dualTable.selection, () => fn.arguments.map<_ArgDefDetails>(a => ({ @@ -92,10 +98,20 @@ export class CreateFunction extends ExecHelper implements _IStatementExecutor { schema: schema, }); + let implementation = compiled; + if (fn.security === 'definer') { + const asOwner = (t: _Transaction, run: () => any) => runAsRole(t, this.owner, run); + implementation = (...a: any[]) => asOwner(executionCtx().transaction, () => compiled(...a)); + const trigger = getTriggerRunner(compiled); + if (trigger) { + (implementation as any).__triggerRunner = (ctx: any, t: _Transaction) => asOwner(t, () => trigger(ctx, t)); + } + } + this.toRegister = { name: fn.name.name, returns, - implementation: compiled, + implementation, args: args.filter(x => x.mode !== 'variadic'), argsVariadic, impure: fn.purity !== 'immutable', @@ -112,8 +128,10 @@ export class CreateFunction extends ExecHelper implements _IStatementExecutor { throw new QueryError(`function ${this.toRegister.name} lready exists with same argument types`, '42723'); } - // ... it must be the same type - if (existing.returns !== returns) { + // ... it must be the same type ("returns trigger" builds a fresh placeholder record + // type each time, so two trigger functions compare by that, not by identity) + const bothTriggers = !!getTriggerRunner(existing.implementation) && isTrigger; + if (existing.returns !== returns && !bothTriggers && !sameReturnType(existing.returns, returns)) { throw new QueryError(`cannot change return type of existing function`, '42P13'); } @@ -128,14 +146,19 @@ export class CreateFunction extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because does not support further rollbacks) - t = t.fullCommit(); + // the creating role owns the function (what SECURITY DEFINER runs as) + this.owner = currentRoleName(t); this.onSchema.registerFunction(this.toRegister, this.replace); - - // new implicit transaction - t = t.fork(); return this.noData(t, 'CREATE'); } } + +/** same declared return type, compared structurally (types are not always interned) */ +function sameReturnType(a: _IType | nil, b: IType | nil): boolean { + if (!a || !b) { + return a === b; + } + const x = a as any, y = b as any; + return x.primary === y.primary && x.name === y.name && (x.of?.name ?? null) === (y.of?.name ?? null); +} diff --git a/src/execution/schema-amends/create-index.ts b/src/execution/schema-amends/create-index.ts index fccff2b0..32eeade0 100644 --- a/src/execution/schema-amends/create-index.ts +++ b/src/execution/schema-amends/create-index.ts @@ -53,16 +53,10 @@ export class CreateIndexExec extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because the index creation does not support further rollbacks) - t = t.fullCommit(); // create index this.onTable .createIndex(t, this.indexDef); - - // new implicit transaction - t = t.fork(); return this.noData(t, 'CREATE'); } } diff --git a/src/execution/schema-amends/create-materialized-view.ts b/src/execution/schema-amends/create-materialized-view.ts index 34716b2f..e78a9539 100644 --- a/src/execution/schema-amends/create-materialized-view.ts +++ b/src/execution/schema-amends/create-materialized-view.ts @@ -33,15 +33,8 @@ export class CreateMaterializedView extends ExecHelper implements _IStatementExe return this.noData(t, 'CREATE'); } - // commit pending data before making changes - // (because does not support further rollbacks) - t = t.fullCommit(); - // view creation this.toRegister.register(); - - // new implicit transaction - t = t.fork(); return this.noData(t, 'CREATE'); } } \ No newline at end of file diff --git a/src/execution/schema-amends/create-policy.ts b/src/execution/schema-amends/create-policy.ts index b239cc10..41faaa3f 100644 --- a/src/execution/schema-amends/create-policy.ts +++ b/src/execution/schema-amends/create-policy.ts @@ -1,7 +1,10 @@ -import { _ISchema, _Transaction, _ITable, _IStatementExecutor, asTable, StatementResult, _IStatement } from '../../interfaces-private'; -import { CreatePolicyStatement, DropPolicyStatement } from 'pgsql-ast-parser'; +import { _ISchema, _Transaction, _ITable, _IStatementExecutor, asTable, StatementResult, _IStatement, QueryError, DataType, nil } from '../../interfaces-private'; +import { CreatePolicyStatement, DropPolicyStatement, Expr } from 'pgsql-ast-parser'; import { ignore } from '../../utils'; import { ExecHelper } from '../exec-utils'; +import { buildValue } from '../../parser/expression-builder'; +import { withSelection } from '../../parser/context'; +import { assertRolesExist } from '../roles'; export class CreatePolicy extends ExecHelper implements _IStatementExecutor { private table: _ITable; @@ -9,13 +12,35 @@ export class CreatePolicy extends ExecHelper implements _IStatementExecutor { constructor({ schema }: _IStatement, private p: CreatePolicyStatement) { super(p); this.table = asTable(schema.getObject(p.table)); - // the predicates are stored raw and compiled lazily at enforcement time + const cmd = p.for ?? 'all'; + if (cmd === 'insert' && p.using) { + throw new QueryError('only WITH CHECK expression allowed for INSERT', '42601'); + } + if ((cmd === 'select' || cmd === 'delete') && p.withCheck) { + throw new QueryError('WITH CHECK cannot be applied to SELECT or DELETE', '42601'); + } + // Postgres binds the predicates at CREATE POLICY: an unknown column, an operator that + // does not exist for the operand types (auth.uid() = ) or a non-boolean + // predicate fails here, not on the first query. Compile them now to fail the same way; + // enforcement recompiles them from the stored AST. + const check = (expr: Expr | nil, clause: string) => { + if (!expr) { + return; + } + const v = withSelection(this.table.selection, () => buildValue(expr)); + if (v.type.primary !== DataType.bool && v.type.primary !== DataType.null) { + throw new QueryError(`argument of POLICY ${clause} must be type boolean, not type ${v.type.name}`, '42804'); + } + }; + check(p.using, 'USING'); + check(p.withCheck, 'WITH CHECK'); ignore(p.using); ignore(p.withCheck); p.roles?.forEach(ignore); } execute(t: _Transaction): StatementResult { + assertRolesExist(t, (this.p.roles ?? []).map(r => r.name)); this.table.createPolicy({ name: this.p.name.name, // postgres default is PERMISSIVE diff --git a/src/execution/schema-amends/create-schema.ts b/src/execution/schema-amends/create-schema.ts index 915cdc2f..6c6c4a5e 100644 --- a/src/execution/schema-amends/create-schema.ts +++ b/src/execution/schema-amends/create-schema.ts @@ -20,17 +20,11 @@ export class CreateSchema extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because does not support further rollbacks) - t = t.fullCommit(); // create schema if (this.toCreate) { this.st.schema.db.createSchema(this.toCreate); } - - // new implicit transaction - t = t.fork(); return this.noData(t, 'CREATE'); } } diff --git a/src/execution/schema-amends/create-sequence.ts b/src/execution/schema-amends/create-sequence.ts index 017cc9b0..eb8f699c 100644 --- a/src/execution/schema-amends/create-sequence.ts +++ b/src/execution/schema-amends/create-sequence.ts @@ -12,15 +12,9 @@ export class ExecuteCreateSequence extends ExecHelper implements _IStatementExec } execute(t: _Transaction) { - // commit pending data before making changes - // (because the index sequence creation does support further rollbacks) - t = t.fullCommit(); // create the sequence this.createSeq(t); - - // new implicit transaction - t = t.fork(); return this.noData(t, 'CREATE'); } diff --git a/src/execution/schema-amends/create-table.ts b/src/execution/schema-amends/create-table.ts index be1cc85e..ae9e9235 100644 --- a/src/execution/schema-amends/create-table.ts +++ b/src/execution/schema-amends/create-table.ts @@ -32,6 +32,8 @@ export class ExecuteCreateTable extends ExecHelper implements _IStatementExecuto type: this.schema.getType(f.dataType), serial: !f.dataType.kind && (f.dataType.name === 'serial' || f.dataType.name === 'bigserial'), }; + // keep the type as written, for information_schema (see ColRef.declaredType) + (nf as any).declaredType = f.dataType; delete (nf as Optional).dataType; fields.push(nf); break; @@ -57,15 +59,11 @@ export class ExecuteCreateTable extends ExecHelper implements _IStatementExecuto execute(t: _Transaction) { - // commit pending data before making changes - // (because the creation does not support further rollbacks) - t = t.fullCommit(); - const partitionOf = this.p.partitionOf; const partitionBy = this.p.partitionBy; // perform creation - checkExistence(this.schema, this.name, this.ifNotExists, () => { + const created = checkExistence(this.schema, this.name, this.ifNotExists, () => { if (partitionOf) { // a partition inherits its parent's columns const parent = asTable(this.schema.getObject(partitionOf.parent)); @@ -73,19 +71,20 @@ export class ExecuteCreateTable extends ExecHelper implements _IStatementExecuto name: c.id!, type: c.type, })); - const child = this.schema.declareTable({ name: this.toDeclare.name, fields }); + const child = this.schema.declareTable({ name: this.toDeclare.name, fields }, false, t); setupChildPartition(parent, child, partitionOf.bound); } else { - const table = this.schema.declareTable(this.toDeclare); + const table = this.schema.declareTable(this.toDeclare, false, t); if (partitionBy) { setupPartitionedParent(table, partitionBy); } } }); - - - // new implicit transaction - t = t.fork(); + if (!created) { + // IF NOT EXISTS on an existing table is a no-op in postgres (a NOTICE): the column + // definitions and their constraints are intentionally unused, not unsupported + ignore(this.p); + } return this.noData(t, 'CREATE'); } } \ No newline at end of file diff --git a/src/execution/schema-amends/create-trigger.ts b/src/execution/schema-amends/create-trigger.ts index 6a59782c..57df2291 100644 --- a/src/execution/schema-amends/create-trigger.ts +++ b/src/execution/schema-amends/create-trigger.ts @@ -3,7 +3,7 @@ import { CreateTriggerStatement } from 'pgsql-ast-parser'; import { buildValue } from '../../parser/expression-builder'; import { ExecHelper } from '../exec-utils'; import { TriggerOp } from '../triggers'; -import { compileTriggerWhen, TriggerContext } from '../plpgsql'; +import { compileTriggerWhen, getTriggerRunner, TriggerContext } from '../plpgsql'; export class CreateTrigger extends ExecHelper implements _IStatementExecutor { private target: _ITable | _IView; @@ -30,6 +30,15 @@ export class CreateTrigger extends ExecHelper implements _IStatementExecutor { } } this.functionSchema = schema.getThisOrSiblingFor(p.execute.function); + // the function is resolved at CREATE TRIGGER, not at the first firing + const fnName = p.execute.function.name; + const fn = this.functionSchema.getFunction(fnName, []); + if (!fn) { + throw new QueryError(`function ${p.execute.function.schema ? p.execute.function.schema + '.' : ''}${fnName}() does not exist`, '42883'); + } + if (!getTriggerRunner(fn.implementation)) { + throw new QueryError(`function ${fnName} must return type trigger`, '42P17'); + } this.events = p.events .filter(e => e.event !== 'truncate') .map(e => e.event as TriggerOp); @@ -50,7 +59,10 @@ export class CreateTrigger extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction): StatementResult { - t = t.fullCommit(); + if (this.p.orReplace) { + // CREATE OR REPLACE TRIGGER (postgres 14+) + this.target.dropTrigger(this.p.name.name, true); + } this.target.createTrigger({ name: this.p.name.name, timing: this.p.timing, @@ -62,7 +74,6 @@ export class CreateTrigger extends ExecHelper implements _IStatementExecutor { updateColumns: this.updateColumns, arguments: this.arguments, }); - t = t.fork(); return this.noData(t, 'CREATE TRIGGER'); } } diff --git a/src/execution/schema-amends/create-view.ts b/src/execution/schema-amends/create-view.ts index 649b7844..027e7992 100644 --- a/src/execution/schema-amends/create-view.ts +++ b/src/execution/schema-amends/create-view.ts @@ -18,6 +18,9 @@ export class CreateView extends ExecHelper implements _IStatementExecutor { // check existence this.existing = asView(this.schema.getObject(p.name, { nullIfNotFound: true })); ignore(p.orReplace); + // WITH (security_invoker = true, security_barrier, ...): accepted; pg-mem evaluates a view's + // query in the caller's context, which is what security_invoker asks for + ignore(p.parameters); this.drop = !!(p.orReplace && this.existing); let view = buildSelect(p.query); @@ -43,9 +46,6 @@ export class CreateView extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because does not support further rollbacks) - t = t.fullCommit(); // drop if needed if (this.existing && this.drop) { @@ -54,9 +54,6 @@ export class CreateView extends ExecHelper implements _IStatementExecutor { // view creation this.toRegister.register(); - - // new implicit transaction - t = t.fork(); return this.noData(t, 'CREATE'); } } diff --git a/src/execution/schema-amends/drop-index.ts b/src/execution/schema-amends/drop-index.ts index 2f44fb81..2eaa5a1f 100644 --- a/src/execution/schema-amends/drop-index.ts +++ b/src/execution/schema-amends/drop-index.ts @@ -22,18 +22,12 @@ export class DropIndex extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because the index sequence creation does support further rollbacks) - t = t.fullCommit(); // alter the sequence for (const idx of this.idx) { idx.onTable.dropIndex(t, idx.name); } - // new implicit transaction - t = t.fork(); - return this.noData(t, 'DROP'); } } diff --git a/src/execution/schema-amends/drop-sequence.ts b/src/execution/schema-amends/drop-sequence.ts index e7f513fc..7ae53506 100644 --- a/src/execution/schema-amends/drop-sequence.ts +++ b/src/execution/schema-amends/drop-sequence.ts @@ -18,18 +18,12 @@ export class DropSequence extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because the index sequence creation does support further rollbacks) - t = t.fullCommit(); // drop the sequence for (const seq of this.seqs) { seq.drop(t); } - // new implicit transaction - t = t.fork(); - return this.noData(t, 'DROP'); } } diff --git a/src/execution/schema-amends/drop-table.ts b/src/execution/schema-amends/drop-table.ts index e4587005..3d9a8e89 100644 --- a/src/execution/schema-amends/drop-table.ts +++ b/src/execution/schema-amends/drop-table.ts @@ -23,18 +23,12 @@ export class DropTable extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because it does not support further rollbacks) - t = t.fullCommit(); // drop table for (const table of this.tables) { table.drop(t, this.cascade); } - // new implicit transaction - t = t.fork(); - return this.noData(t, 'DROP'); } } diff --git a/src/execution/schema-amends/drop-type.ts b/src/execution/schema-amends/drop-type.ts index c23ecba9..47104308 100644 --- a/src/execution/schema-amends/drop-type.ts +++ b/src/execution/schema-amends/drop-type.ts @@ -18,18 +18,12 @@ export class DropType extends ExecHelper implements _IStatementExecutor { } execute(t: _Transaction) { - // commit pending data before making changes - // (because the index sequence creation does support further rollbacks) - t = t.fullCommit(); // drop the sequence for (const seq of this.types) { seq.drop(t); } - // new implicit transaction - t = t.fork(); - return this.noData(t, 'DROP'); } } diff --git a/src/execution/statement-exec.ts b/src/execution/statement-exec.ts index 9ed0d688..bf0019e6 100644 --- a/src/execution/statement-exec.ts +++ b/src/execution/statement-exec.ts @@ -15,7 +15,7 @@ import { CommitExecutor, RollbackExecutor, BeginStatementExec, SavepointExecutor import { TruncateTable } from './records-mutations/truncate-table'; import { ShowExecutor } from './show'; import { SetExecutor } from './set'; -import { CreateRoleExecutor, DropRoleExecutor, SetRoleExecutor, ResetExecutor } from './roles'; +import { CreateRoleExecutor, DropRoleExecutor, SetRoleExecutor, ResetExecutor, assertRolesExist } from './roles'; import { CreatePolicy, DropPolicy } from './schema-amends/create-policy'; import { CreateTrigger } from './schema-amends/create-trigger'; import { CreateEnum } from './schema-amends/create-enum'; @@ -35,6 +35,9 @@ import { CreateCompositeType } from './schema-amends/create-composite-type'; import { InsteadOfView } from './records-mutations/instead-of'; import { MergeExec } from './records-mutations/merge'; import { hasInsteadOf, TriggerOp } from './triggers'; +import { isSchemaStatement } from '../persistence'; +import { captureSchema } from '../schema-snapshot'; +import { withRiQueue } from './ri-queue'; import { _IView, _ITable } from '../interfaces-private'; const detailsIncluded = Symbol('errorDetailsIncluded'); @@ -62,7 +65,7 @@ export class StatementExec implements _IStatement { private executor?: _IStatementExecutor; private checkAstCoverage?: (() => void); - constructor(readonly schema: _ISchema, private statement: Statement, private pAsSql: string | nil, private parameters?: Parameter[]) { + constructor(readonly schema: _ISchema, readonly statement: Statement, private pAsSql: string | nil, private parameters?: Parameter[]) { } onExecuted(callback: OnStatementExecuted): void { @@ -163,14 +166,29 @@ export class StatementExec implements _IStatement { case 'create function': return new CreateFunction(this, p); case 'drop function': - return new SimpleExecutor(p, () => this.schema.dropFunction(p), 'DROP'); + return new SimpleExecutor(p, () => { + ignore(p.cascade); + this.schema.dropFunction(p); + }, 'DROP'); case 'do': return new DoStatementExec(this, p); case 'comment': return new Comment(this, p); - case 'raise': case 'grant': case 'revoke': + // no privilege system: privileges are not recorded, but the grantees and (for + // tables) the objects must exist, as postgres checks + return new SimpleExecutor(p, t => { + assertRolesExist(t, ((p as any).to ?? (p as any).from ?? []).map((r: any) => r?.name)); + const on = (p as any).on; + if (on?.type === 'table') { + for (const n of on.names ?? []) { + this.schema.getThisOrSiblingFor(n).getObject(n); + } + } + ignore(p); + }); + case 'raise': case 'alter role': case 'alter default privileges': // pg-mem has no privilege/role-config system: parse & ignore (dumps, RLS setup) @@ -246,6 +264,22 @@ export class StatementExec implements _IStatement { }, 'DROP TRIGGER'); case 'alter index': return new AlterIndex(this, p); + case 'drop view': + case 'drop materialized view': + // (materialized views are plain views in pg-mem) + return new SimpleExecutor(p, t => { + ignore(p.cascade); + for (const n of p.names) { + const obj = this.schema.getThisOrSiblingFor(n).getObject(n, { nullIfNotFound: !!p.ifExists }); + if (!obj) { + continue; // IF EXISTS + } + if (obj.type !== 'view') { + throw new QueryError(`"${n.name}" is not a view`, '42809'); + } + (obj as _IView).drop(t); + } + }, 'DROP VIEW'); default: throw NotSupported.never(p, 'statement type'); } @@ -324,7 +358,11 @@ export class StatementExec implements _IStatement { if (!this.executor) { throw new Error('Statement not prepared') } - const result = this.executor.execute(t); + if (isSchemaStatement(this.statement) || this.statement.type === 'do') { + // DDL is transactional: remember the schema so a rollback can put it back + t.checkpointSchema(() => captureSchema(this.schema.db)); + } + const result = withRiQueue(() => this.executor!.execute(t)); // post-execution for (const s of this.onExecutedCallbacks) { diff --git a/src/execution/transaction-statements.ts b/src/execution/transaction-statements.ts index d673eb2d..a004a286 100644 --- a/src/execution/transaction-statements.ts +++ b/src/execution/transaction-statements.ts @@ -67,7 +67,7 @@ export class BeginStatementExec extends ExecHelper implements _IStatementExecuto } execute(t: _Transaction): StatementResult { - t = t.fork(); + t = t.fork(true); return this.noData(t, 'BEGIN'); } } diff --git a/src/functions/date.ts b/src/functions/date.ts index 11dd2224..776b2388 100644 --- a/src/functions/date.ts +++ b/src/functions/date.ts @@ -2,7 +2,7 @@ import { FunctionDefinition } from '../interfaces'; import { utc, fromParts, startOf, parseFormat } from '../datatypes/date-utils'; import { Interval } from 'pgsql-ast-parser'; import { DataType, QueryError } from '../interfaces-private'; -import { nullIsh } from '../utils'; +import { nullIsh, transactionNow } from '../utils'; import { dateToChar, intervalToChar, numberToChar } from './to-char'; @@ -22,8 +22,17 @@ export const dateFunctions: FunctionDefinition[] = [ return ret; } }, + // now() and transaction_timestamp() are the transaction's start time, constant for its whole + // lifetime (so a trigger's `new.updated_at = now()` equals the statement's now()); + // clock_timestamp() is the wall clock + ...['now', 'transaction_timestamp', 'statement_timestamp'].map(name => ({ + name, + returns: DataType.timestamptz, + impure: true, + implementation: () => transactionNow(), + })), { - name: 'now', + name: 'clock_timestamp', returns: DataType.timestamptz, impure: true, implementation: () => new Date(), diff --git a/src/functions/string.ts b/src/functions/string.ts index c5056614..e3c1fd60 100644 --- a/src/functions/string.ts +++ b/src/functions/string.ts @@ -128,8 +128,9 @@ export const stringFunctions: FunctionDefinition[] = [ implementation: (x: string) => x?.toUpperCase(), }, { + // concat(VARIADIC "any"): every argument, the first included, may be of any type name: 'concat', - args: [DataType.text], + args: [], argsVariadic: DataType.text, returns: DataType.text, allowNullArguments: true, diff --git a/src/interfaces-private.ts b/src/interfaces-private.ts index 8feed381..be6820b6 100644 --- a/src/interfaces-private.ts +++ b/src/interfaces-private.ts @@ -55,7 +55,7 @@ export interface _ISchema extends ISchema { getTable(table: string, nullIfNotFound?: boolean): _ITable; tablesCount(t: _Transaction): number; listTables(t?: _Transaction): Iterable<_ITable>; - declareTable(table: Schema, noSchemaChange?: boolean): _ITable; + declareTable(table: Schema, noSchemaChange?: boolean, inTransaction?: _Transaction): _ITable; createSequence(t: _Transaction, opts: CreateSequenceOptions | nil, name: QName | nil): _ISequence; /** Get functions matching this overload */ resolveFunction(name: string | QName, args: IValue[], forceOwn?: boolean): _FunctionDefinition | nil; @@ -155,15 +155,27 @@ export type _ArgDefDetails = ArgDefDetails & { export interface _Transaction { readonly isChild: boolean; - /** Create a new transaction within this transaction */ - fork(): _Transaction; + /** true inside a BEGIN … COMMIT/ROLLBACK block (this transaction or an ancestor was opened by BEGIN) */ + readonly inExplicitBlock: boolean; + /** transaction start: the value of now() / current_timestamp for its whole lifetime */ + readonly startedAt: Date; + /** set when a statement failed inside an explicit block: Postgres refuses everything but COMMIT/ROLLBACK until the block ends */ + aborted: boolean; + /** Create a new transaction within this transaction (explicit = opened by BEGIN) */ + fork(explicit?: boolean): _Transaction; /** Commit this transaction (returns the parent transaction) */ commit(): _Transaction; /** Commits this transaction and all underlying transactions */ fullCommit(): _Transaction; rollback(): _Transaction; - /** Capture the current state under a named savepoint */ + /** Capture the current state under a named savepoint (the schema is captured lazily, by the next DDL) */ savepoint(name: string): void; + /** Record the schema before this transaction's first DDL, so a rollback can restore it */ + checkpointSchema(capture: () => () => void): void; + /** Roll back the schema of this transaction and every enclosing one (a failed call) */ + discardAll(): void; + /** The BEGIN block this transaction belongs to, if any */ + readonly explicitBlock: _Transaction | null; /** Restore the state captured by a named savepoint (keeping the savepoint) */ rollbackTo(name: string): void; /** Discard a named savepoint (and any established after it) */ @@ -387,6 +399,13 @@ export interface _IDb extends IMemoryDb { readonly options: MemoryDbOptions; readonly public: _ISchema; readonly data: _Transaction; + /** + * The transaction block a BEGIN left open at the end of a query call. + * Postgres keeps a transaction open across round-trips until COMMIT/ROLLBACK; pg-mem used to + * commit at the end of every call, so `begin` / `delete` / `rollback` sent separately (as every + * driver does) persisted the delete. The next call resumes this transaction instead of forking root. + */ + sessionTx: _Transaction | null; readonly searchPath: ReadonlyArray; /** session-scoped named prepared statements (SQL-level PREPARE / EXECUTE); * each entry is a runner that binds args and executes against a transaction */ @@ -446,6 +465,10 @@ export interface _ITable extends IMemoryTable, _RelationBase { readonly rls: TableRls; createPolicy(policy: Policy): void; dropPolicy(name: string, ifExists: boolean): void; + /** every constraint on this table (FKs, uniques, checks, ...) */ + listConstraints?(): Iterable<_IConstraint>; + /** foreign keys on any table that reference this one (only those through `viaColumns` when given) */ + referencingForeignKeys?(viaColumns?: string[]): _IForeignKeyRef[]; setRowLevelSecurity(action: 'enable' | 'disable' | 'force' | 'no force'): void; /** Triggers attached to this table */ readonly triggers: TableTriggers; @@ -525,6 +548,14 @@ export interface _Column { onDrop(sub: DropHandler): ISubscription; } +/** The parts of a foreign key that dependency checks need. */ +export interface _IForeignKeyRef { + readonly name: string; + readonly tableName: string; + readonly foreignColumns: string[]; + uninstall(t: _Transaction): void; +} + export interface CreateIndexDef { ifNotExists?: boolean; columns: CreateIndexColDef[]; diff --git a/src/parser/expression-builder.ts b/src/parser/expression-builder.ts index 5d4c9117..a7c6476c 100644 --- a/src/parser/expression-builder.ts +++ b/src/parser/expression-builder.ts @@ -1,10 +1,10 @@ import { _ISelection, IValue, _IType, _ISchema, _IAlias, _Transaction } from '../interfaces-private'; import { currentRoleName, sessionRoleName } from '../execution/roles'; -import { buildLikeMatcher, nullIsh, hasNullish, intervalToSec, parseTime, asSingleQName, colToStr, executionCtx, ignore } from '../utils'; +import { buildLikeMatcher, nullIsh, hasNullish, intervalToSec, parseTime, asSingleQName, colToStr, executionCtx, ignore, transactionNow } from '../utils'; import { DataType, CastError, QueryError, NotSupported, nil, ColumnNotFound } from '../interfaces'; import hash from 'object-hash'; import { Value, Evaluator } from '../evaluator'; -import { Types, isNumeric, reconciliateTypes, ArrayType, RecordCol } from '../datatypes'; +import { Types, isNumeric, reconciliateTypes, ArrayType, RecordCol, crossesTypeCategory } from '../datatypes'; import { Expr, ExprBinary, UnaryOperator, ExprCase, ExprWhen, ExprMember, ExprArrayIndex, ExprTernary, BinaryOperator, SelectStatement, ExprValueKeyword, ExprExtract, Interval, ExprOverlay, ExprSubstring, ExprPosition, ExprCall } from 'pgsql-ast-parser'; import lru from 'lru-cache'; import { aggregationFunctions, getAggregator } from '../transforms/aggregation'; @@ -39,6 +39,11 @@ export function uncache(data: _ISelection) { builtLru.del(data); } +/** Drops every cached build: a build can embed schema state (e.g. another table's policies in a subquery) */ +export function uncacheAll() { + builtLru.reset(); +} + function _buildValue(val: Expr): IValue { // cache expressions build (they almost are always rebuilt several times in a row) const data = buildCtx().selection; @@ -123,6 +128,11 @@ function _buildValueReal(val: Expr): IValue { ignore(val.value); return new Evaluator(Types.bigint, null, val.valueText, null, val.valueText); } + if (val.value > 2147483647 || val.value < -2147483648) { + // postgres types an integer literal that does not fit int4 as bigint, so that + // assigning it to an integer column is "integer out of range" + return new Evaluator(Types.bigint, null, String(val.value), null, String(val.value)); + } return Value.number(val.value, Types.integer); case 'call': return _buildCall(val); @@ -307,11 +317,14 @@ function buildKeyword(kw: ExprValueKeyword, args: Expr[]): IValue { return Value.constant(Types.text(), 'pg_mem'); case 'current_schema': return Value.constant(Types.text(), 'public'); + // evaluated per execution, not at compile time (a cached plan must not freeze the clock), + // from the transaction start like now(); current_date is that instant's date case 'current_date': - return Value.constant(Types.date, new Date()); + return transactionClock(Types.timestamptz()).cast(Types.date); case 'current_timestamp': + return transactionClock(Types.timestamptz()); case 'localtimestamp': - return Value.constant(Types.timestamp(), new Date()); + return transactionClock(Types.timestamptz()).cast(Types.timestamp()); case 'localtime': case 'current_time': throw new NotSupported('"date" data type, please file an issue in https://github.com/oguimbal/pg-mem if you need it !'); @@ -322,6 +335,10 @@ function buildKeyword(kw: ExprValueKeyword, args: Expr[]): IValue { } } +function transactionClock(type: _IType): IValue { + return new Evaluator(type, null, Math.random().toString(), [], () => transactionNow(), { unpure: true }); +} + function buildUnary(op: UnaryOperator, operand: Expr) { const expr = _buildValue(operand); @@ -364,8 +381,26 @@ export function buildSubqueryArray(op: Expr): IValue { function buildIn(left: Expr, array: Expr, inclusive: boolean): IValue { let leftValue = _buildValue(left); + if (array.type === 'list') { + // `x IN (a, b)` is `x = a OR x = b`: every element must be comparable with x, + // so `uuid_col IN (text_col)` fails like `uuid_col = text_col` does + reconciliateTypes([leftValue, ...array.expressions.map(e => _buildValue(e))]); + } // `x IN (subquery)` needs the subquery's rows as a list, not a scalar let rightValue = isSubqueryNode(array) ? buildSelectAsArray(array as any) : _buildValue(array); + if (isSubqueryNode(array)) { + // `x IN (select col ...)` compares x with the column's type: `auth.uid() in (select )` + // is "operator does not exist: uuid = text", like the list and = forms + const col = (rightValue.type as ArrayType).of; + const leftUnknown = leftValue.isConstantLiteral && leftValue.type.primary === DataType.text; + if (col && !leftUnknown && crossesTypeCategory(leftValue.type, col)) { + throw new QueryError(`operator does not exist: ${leftValue.type.name} = ${col.name}`, '42883'); + } + } + if (array.type !== 'list' && rightValue.type.primary !== DataType.list && rightValue.type.primary !== DataType.array) { + // `x IN (y)` - the parser drops the parens of a one-element list + reconciliateTypes([leftValue, rightValue]); + } return Value.in(leftValue, rightValue, inclusive); } @@ -421,6 +456,17 @@ export function buildBinaryValue(leftValue: IValue, op: BinaryOperator, rightVal // resolve a custom operator registered on the schema (ranges, extensions, ...) const resolveCustom = () => { const { schema } = buildCtx(); + // postgres has `text || anynonarray` and `anynonarray || text`: the other side goes + // through its text output function ('n=' || 1, 'at ' || now()) + if (op === '||') { + const isStr = (v: IValue) => v.type.primary === DataType.text || v.type.primary === DataType.citext; + const nonArray = (v: IValue) => v.type.primary !== DataType.array && v.type.primary !== DataType.list && v.type.primary !== DataType.jsonb && v.type.primary !== DataType.null; + if (isStr(leftValue) && !isStr(rightValue) && nonArray(rightValue) && !rightValue.isConstantLiteral) { + rightValue = rightValue.cast(Types.text()); + } else if (isStr(rightValue) && !isStr(leftValue) && nonArray(leftValue) && !leftValue.isConstantLiteral) { + leftValue = leftValue.cast(Types.text()); + } + } const resolved = schema.resolveOperator(op, leftValue, rightValue); if (!resolved) { throw new QueryError(`operator does not exist: ${leftValue.type.name} ${op} ${rightValue.type.name}`, '42883'); @@ -510,6 +556,12 @@ export function buildBinaryValue(leftValue: IValue, op: BinaryOperator, rightVal case 'ILIKE': case 'NOT LIKE': case 'NOT ILIKE': + // LIKE is text ~~ text: a typed non-string operand has no operator (integer ~~ unknown) + for (const v of [leftValue, rightValue]) { + if (!(v.isConstantLiteral && v.type.primary === DataType.text) && crossesTypeCategory(v.type, Types.text()) && !rightValue.isAny) { + throw new QueryError(`operator does not exist: ${leftValue.type.name} ~~ ${rightValue.type.name}`, '42883'); + } + } expectBoth(Types.text()); const caseSenit = op === 'LIKE' || op === 'NOT LIKE'; const not = op === 'NOT ILIKE' || op === 'NOT LIKE'; diff --git a/src/parser/function-call.ts b/src/parser/function-call.ts index a4a95409..28b709fc 100644 --- a/src/parser/function-call.ts +++ b/src/parser/function-call.ts @@ -1,5 +1,5 @@ import { IValue, _IType, _ISelection, _ISchema, _IDb, _Transaction } from '../interfaces-private'; -import { Types, ArrayType } from '../datatypes'; +import { Types, ArrayType, crossesTypeCategory } from '../datatypes'; import { toJsonValue } from '../datatypes/json-numbers'; import { QueryError, NotSupported, nil, DataType } from '../interfaces'; import { Evaluator } from '../evaluator'; @@ -264,6 +264,12 @@ export function buildCall(name: string | QName, args: IValue[]): IValue { } throw new QueryError(`COALESCE types ${a.name} and ${b.type.name} cannot be matched`, '42804'); }, args[0].type); + for (const x of args) { + // a typed argument from another category cannot be matched (COALESCE(uuid_col, text_col)) + if (!(x.isConstantLiteral && x.type.primary === DataType.text) && crossesTypeCategory(x.type, type)) { + throw new QueryError(`COALESCE types ${type.name} and ${x.type.name} cannot be matched`, '42804'); + } + } args = args.map(x => x.cast(type!)); get = (...args: any[]) => args.find(x => !nullIsh(x)); break; diff --git a/src/schema-snapshot.ts b/src/schema-snapshot.ts new file mode 100644 index 00000000..177acb2b --- /dev/null +++ b/src/schema-snapshot.ts @@ -0,0 +1,163 @@ +import { _IDb } from './interfaces-private'; + +const SYSTEM_SCHEMAS = new Set(['pg_catalog', 'information_schema']); + +/** + * Transactional DDL. + * + * Row data lives in immutable maps inside the transaction, so rolling it back is just dropping the + * transaction. Schema objects (tables, columns, constraints, indexes, policies, triggers, functions, + * types) are mutable objects outside it. To make DDL roll back like it does in postgres - a failed + * migration leaves no half-created tables - the schema metadata is captured before the first DDL of + * a transaction and put back if that transaction rolls back. + * + * The capture is a copy of each schema object's own properties, with containers (Map, Set, Array, + * plain object) copied a few levels deep and class instances kept by reference - those that are + * themselves mutable schema objects are captured in their own right. Objects created after the + * capture simply drop out of the restored registries. + */ +export function captureSchema(db: _IDb): () => void { + const restorers: (() => void)[] = []; + const seen = new Set(); + const capture = (o: any, skip: string[] = []) => { + if (!o || typeof o !== 'object' || seen.has(o)) { + return; + } + seen.add(o); + restorers.push(captureProps(o, skip)); + }; + + // never part of the schema: row data and the session's open transaction + capture(db, ['data', 'sessionTx', 'schemaVersion', 'handlers']); + for (const schema of db.listSchemas()) { + const s = schema as any; + // the built-in catalogues (and their hundreds of functions) are not changed by user DDL; + // skipping them keeps the capture cheap enough to take on every DDL transaction + if (SYSTEM_SCHEMAS.has(s.name)) { + continue; + } + capture(s, ['fns', 'ops', 'interceptors', 'db']); + for (const resolver of [s.fns, s.ops]) { + if (resolver?.snapshot) { + restorers.push(resolver.snapshot()); + } + } + for (const rel of s.relsByNameCas?.values?.() ?? []) { + capture(rel, ['ownerSchema', 'db']); + const r = rel as any; + if (r.type !== 'table') { + continue; + } + capture(r.columnMgr); + capture(r.columnMgr?.map); + for (const col of r.columnMgr?.map?.values?.() ?? []) { + capture(col, ['table']); + } + capture(r.rls); + capture(r.triggers); + for (const c of r.listConstraints?.() ?? []) { + capture(c); + capture(c.wrapped); + } + } + } + + return () => { + for (const r of restorers) { + r(); + } + for (const schema of db.listSchemas()) { + for (const rel of (schema as any).relsByNameCas?.values?.() ?? []) { + rel.columnMgr?.invalidateColumns?.(); + rel.selection?.rebuild?.(); + } + } + db.onSchemaChange(); + }; +} + +/** a captured value: containers keep their identity and remember their contents */ +type Snap = { ref: any; kind: 'map' | 'set' | 'array' | 'object'; items: [any, Snap][] } | { value: any }; + +function snap(v: any, depth: number): Snap { + // a frozen value cannot have changed: keep it by reference + if (depth < 0 || !v || typeof v !== 'object' || Object.isFrozen(v)) { + return { value: v }; + } + if (v instanceof Map) { + return { ref: v, kind: 'map', items: [...v].map(([k, x]) => [k, snap(x, depth - 1)]) }; + } + if (v instanceof Set) { + return { ref: v, kind: 'set', items: [...v].map(x => [null, snap(x, depth - 1)]) }; + } + if (Array.isArray(v)) { + return { ref: v, kind: 'array', items: v.map(x => [null, snap(x, depth - 1)]) }; + } + if (Object.getPrototypeOf(v) === Object.prototype) { + return { ref: v, kind: 'object', items: Object.keys(v).map(k => [k, snap(v[k], depth - 1)]) }; + } + return { value: v }; +} + +/** + * Put a captured value back, refilling the original container instances in place - other code + * holds references to them (e.g. ColumnManager binds get/has to its Map), so swapping in copies + * would orphan those. + */ +function unsnap(s: Snap): any { + if ('value' in s) { + return s.value; + } + const items = s.items.map(([k, x]) => [k, unsnap(x)] as [any, any]); + switch (s.kind) { + case 'map': + s.ref.clear(); + for (const [k, v] of items) { s.ref.set(k, v); } + break; + case 'set': + s.ref.clear(); + for (const [, v] of items) { s.ref.add(v); } + break; + case 'array': + s.ref.length = 0; + s.ref.push(...items.map(([, v]) => v)); + break; + case 'object': { + const keep = new Set(items.map(([k]) => k)); + for (const k of Object.keys(s.ref)) { + if (!keep.has(k) && Object.getOwnPropertyDescriptor(s.ref, k)?.configurable) { + delete s.ref[k]; + } + } + for (const [k, v] of items) { + if (s.ref[k] !== v && Object.getOwnPropertyDescriptor(s.ref, k)?.writable !== false) { + s.ref[k] = v; + } + } + break; + } + } + return s.ref; +} + +function captureProps(o: any, skip: string[]): () => void { + if (o instanceof Map || o instanceof Set) { + const whole = snap(o, 3); + return () => { unsnap(whole); }; + } + const keys = Object.keys(o).filter(k => !skip.includes(k)); + const saved = new Map(keys.map(k => [k, snap(o[k], 3)])); + return () => { + for (const k of Object.keys(o)) { + if (!skip.includes(k) && !saved.has(k) && Object.getOwnPropertyDescriptor(o, k)?.configurable) { + delete o[k]; + } + } + for (const [k, v] of saved) { + const val = unsnap(v); + if (o[k] !== val && Object.getOwnPropertyDescriptor(o, k)?.writable !== false) { + o[k] = val; + } + } + }; +} diff --git a/src/schema/information-schema/columns-list.ts b/src/schema/information-schema/columns-list.ts index 60ac0058..5e275efb 100644 --- a/src/schema/information-schema/columns-list.ts +++ b/src/schema/information-schema/columns-list.ts @@ -1,6 +1,7 @@ import { _ITable, _ISelection, IValue, _IIndex, _IDb, IndexKey, setId, _Transaction, _ISchema } from '../../interfaces-private'; import { Schema, nil } from '../../interfaces'; -import { toSql } from 'pgsql-ast-parser'; +import { toSql, DataTypeDef } from 'pgsql-ast-parser'; +import { DataType } from '../../interfaces'; import { Types } from '../../datatypes'; import { TableIndex } from '../table-index'; import { ReadOnlyTable } from '../readonly-table'; @@ -12,10 +13,55 @@ const IS_SCHEMA = Symbol('_is_colmun'); * Only real tables carry ColRefs; views and function-call tables expose values without column * definitions, so callers must tolerate nil rather than assume a table. */ +const BUILTIN = new Set(Object.values(DataType)); + +/** Postgres' internal (pg_type) name for a type, as udt_name reports it. */ +function udtName(type: IValue['type'], declared: DataTypeDef | nil): string { + const n = (declared as any)?.name?.toLowerCase() as string | undefined; + switch (type.primary) { + case DataType.integer: return n === 'smallint' || n === 'int2' || n === 'smallserial' ? 'int2' : 'int4'; + case DataType.bigint: return 'int8'; + case DataType.float: return n === 'real' || n === 'float4' ? 'float4' : 'float8'; + case DataType.decimal: return 'numeric'; + case DataType.text: + if (n === 'char' || n === 'character' || n === 'bpchar') return 'bpchar'; + return (type as any).len || n === 'varchar' || n === 'character varying' ? 'varchar' : 'text'; + case DataType.array: + return '_' + udtName((type as any).of, declared?.kind === 'array' ? declared.arrayOf : null); + } + // built-ins are named by their primary (timestamptz, not 'timestamp with time zone'); enums/domains by name + return BUILTIN.has(type.primary) ? type.primary : (type as any).name ?? type.primary; +} + +/** information_schema.columns.data_type: the SQL-standard name, 'ARRAY', or 'USER-DEFINED'. */ +function sqlTypeName(type: IValue['type'], declared: DataTypeDef | nil): string { + if (type.primary === DataType.array) { + return 'ARRAY'; + } + if (type.primary === DataType.citext || !BUILTIN.has(type.primary)) { + return 'USER-DEFINED'; + } + switch (udtName(type, declared)) { + case 'int2': return 'smallint'; + case 'int4': return 'integer'; + case 'int8': return 'bigint'; + case 'float4': return 'real'; + case 'float8': return 'double precision'; + case 'bool': return 'boolean'; + case 'bpchar': return 'character'; + case 'varchar': return 'character varying'; + case 'timestamptz': return 'timestamp with time zone'; + case 'timestamp': return 'timestamp without time zone'; + case 'timetz': return 'time with time zone'; + case 'time': return 'time without time zone'; + } + return udtName(type, declared); +} + function columnRef( table: _ITable, columnName: string | nil -): { notNull?: boolean; default?: IValue | nil } | nil { +): { notNull?: boolean; default?: IValue | nil; declaredType?: DataTypeDef | nil } | nil { if (!columnName) { return null; } @@ -45,6 +91,10 @@ function defaultExpressionOf(table: _ITable, columnName: string | nil): string | if (!ast) { return null; } + const pg = pgExprText(ast); + if (pg !== null) { + return pg; + } try { // toSql renders defensively — `now()` comes out as `(now () )`. Postgres reports `now()`, and // consumers compare these strings, so collapse the padding and drop one layer of wrapping @@ -64,6 +114,71 @@ function defaultExpressionOf(table: _ITable, columnName: string | nil): string | } } +const pgQuote = (s: string) => `'${s.replace(/'/g, "''")}'`; +const KEYWORD_TEXT: Record = { + current_date: 'CURRENT_DATE', current_timestamp: 'CURRENT_TIMESTAMP', localtimestamp: 'LOCALTIMESTAMP', + current_time: 'CURRENT_TIME', localtime: 'LOCALTIME', current_user: 'CURRENT_USER', current_role: 'CURRENT_ROLE', + session_user: 'SESSION_USER', user: 'CURRENT_USER', +}; + +/** + * A default expression as postgres' pg_get_expr prints it: literals as written (0.30, not 0.3), + * negative numbers quoted ('-1'::integer), a binary expression wrapped once - (now() + '2 days'::interval) + * - with bare operands, ARRAY[...], CURRENT_DATE. Null for shapes it does not know, which then go + * through the generic toSql rendering. + */ +function pgExprText(e: any): string | null { + const operand = (x: any) => pgExprText(x); + switch (e?.type) { + case 'string': + return pgQuote(e.value); + case 'integer': + // negative, or too big for int4 (then typed bigint): postgres prints it quoted + return e.value < 0 || e.value > 2147483647 ? pgQuote(String(e.valueText ?? e.value)) : String(e.valueText ?? e.value); + case 'numeric': { + const txt = e.raw ?? e.valueText ?? String(e.value); + return txt.startsWith('-') ? pgQuote(txt) : txt; + } + case 'boolean': + return e.value ? 'true' : 'false'; + case 'null': + return 'NULL'; + case 'keyword': + return KEYWORD_TEXT[e.keyword] ?? null; + case 'call': { + const args = (e.args ?? []).map(operand); + if (args.some((a: string | null) => a === null) || e.distinct || e.orderBy || e.filter || e.over) { + return null; + } + const fn = (e.function.schema ? e.function.schema + '.' : '') + e.function.name; + return `${fn}(${args.join(', ')})`; + } + case 'cast': { + const inner = operand(e.operand); + return inner === null ? null : `${inner}::${toSql.dataType(e.to as any)}`; + } + case 'binary': { + const l = operand(e.left), r = operand(e.right); + return l === null || r === null ? null : `(${l} ${e.op} ${r})`; + } + case 'unary': { + const v = operand(e.operand); + if (v === null) { + return null; + } + if (e.op === '-' && /^[\d.]+$/.test(v)) { + return pgQuote('-' + v); + } + return e.op === 'NOT' ? `(NOT ${v})` : `(${e.op}${v})`; + } + case 'array': { + const items = (e.expressions ?? []).map(operand); + return items.some((a: string | null) => a === null) ? null : `ARRAY[${items.join(', ')}]`; + } + } + return null; +} + /** Remove one paren pair only when it wraps the entire expression. */ function stripWrappingParens(sql: string): string { if (!sql.startsWith('(') || !sql.endsWith(')')) { @@ -179,14 +294,15 @@ export class ColumnsListSchema extends ReadOnlyTable implements _ITable { // column refs (views, function-call tables) still fall back to the permissive answer. is_nullable: columnRef(table, t.id)?.notNull ? 'NO' : 'YES', column_default: defaultExpressionOf(table, t.id), - data_type: t.type.primary, // <== todo + data_type: sqlTypeName(t.type, columnRef(table, t.id)?.declaredType), + character_maximum_length: (t.type as any).len ?? (udtName(t.type, columnRef(table, t.id)?.declaredType) === 'bpchar' ? 1 : null), numeric_precision: null, // <== todo numeric_precision_radix: null, // <== todo numeric_scale: null, // <== todo udt_catalog: 'pgmem', udt_schema: 'pg_catalog', - udt_name: t.type.primary, // <== todo + udt_name: udtName(t.type, columnRef(table, t.id)?.declaredType), dtd_identifier: i, // <== todo @@ -209,7 +325,7 @@ export class ColumnsListSchema extends ReadOnlyTable implements _ITable { } getIndex(forValue: IValue): _IIndex | nil { - if (forValue.id === 'table_name') { + if (forValue?.id === 'table_name') { return new TableIndex(this, forValue); } return null; diff --git a/src/schema/overload-resolver.ts b/src/schema/overload-resolver.ts index 86a54bb5..fa16ac07 100644 --- a/src/schema/overload-resolver.ts +++ b/src/schema/overload-resolver.ts @@ -1,5 +1,5 @@ import { _IType, _ArgDefDetails, nil, DataType, IValue } from '../interfaces-private'; -import { Types } from '../datatypes'; +import { Types, crossesTypeCategory } from '../datatypes'; import { it } from '../utils'; import { QueryError } from '../interfaces'; @@ -31,6 +31,17 @@ export class OverloadResolver { return [...ovr.all()]; } + /** every registered overload, to put the resolver back as it is now (transactional DDL) */ + snapshot(): () => void { + const all = [...this.byName.keys()].flatMap(n => this.getOverloads(n)); + return () => { + this.byName = new Map(); + for (const v of all) { + this.add(v, true); + } + }; + } + remove(value: T) { this.byName.get(value.name)?.unindex(value); } @@ -141,7 +152,8 @@ class OverloadNode { } // handle variadic args - if (this.leaf && this.leaf.argsVariadic && this.compatible(arg, this.leaf.argsVariadic)) { + // (the variadic builtins - concat, concat_ws, format - take VARIADIC "any" in postgres) + if (this.leaf && this.leaf.argsVariadic && this.compatible(arg, this.leaf.argsVariadic, true)) { return this.leaf; } @@ -149,12 +161,18 @@ class OverloadNode { return null; } - private compatible(givenArg: IValue, expectedArg: _IType) { + private compatible(givenArg: IValue, expectedArg: _IType, anyCategory = false) { if (givenArg.type === expectedArg) { return true; } - return givenArg.isConstantLiteral - ? givenArg.type.canCast(expectedArg) - : givenArg.type.canConvertImplicit(expectedArg) ?? givenArg.type.canCast(expectedArg); + if (givenArg.isConstantLiteral) { + return givenArg.type.canCast(expectedArg); + } + // a typed argument never crosses a type category implicitly: lower(uuid_col) is + // "function lower(uuid) does not exist" in postgres + if (!anyCategory && crossesTypeCategory(givenArg.type, expectedArg)) { + return false; + } + return givenArg.type.canConvertImplicit(expectedArg) ?? givenArg.type.canCast(expectedArg); } } \ No newline at end of file diff --git a/src/schema/pg-catalog/binary-operators.ts b/src/schema/pg-catalog/binary-operators.ts index 180285b1..a7c4f5bb 100644 --- a/src/schema/pg-catalog/binary-operators.ts +++ b/src/schema/pg-catalog/binary-operators.ts @@ -158,16 +158,44 @@ function registerNumericOperators(schema: _ISchema) { } +/** an elapsed time as postgres' interval subtraction justifies it: days, then h/m/s/ms, one sign */ +function msToInterval(ms: number) { + const sign = ms < 0 ? -1 : 1; + let rest = Math.abs(ms); + const days = Math.floor(rest / 86400_000); rest -= days * 86400_000; + const hours = Math.floor(rest / 3600_000); rest -= hours * 3600_000; + const minutes = Math.floor(rest / 60_000); rest -= minutes * 60_000; + const seconds = Math.floor(rest / 1000); rest -= seconds * 1000; + const out: any = {}; + if (days) out.days = sign * days; + if (hours) out.hours = sign * hours; + if (minutes) out.minutes = sign * minutes; + if (seconds) out.seconds = sign * seconds; + if (rest) out.milliseconds = sign * rest; + return out; +} + function registerDatetimeOperators(schema: _ISchema) { - // ======= date "-" date ======= + // ======= date "-" date = integer (days) ======= schema.registerOperator({ operator: '-', commutative: false, left: Types.date, right: Types.date, - returns: Types.interval, + returns: Types.integer, implementation: (a, b) => utc(a).diff(utc(b), 'days'), }) + // ======= timestamp(tz) "-" timestamp(tz) = interval, as days + time (never months) ======= + for (const tt of [Types.timestamptz(), Types.timestamp()]) { + schema.registerOperator({ + operator: '-', + commutative: false, + left: tt, + right: tt, + returns: Types.interval, + implementation: (a: Date, b: Date) => msToInterval(a.getTime() - b.getTime()), + }); + } // ======= date/time "+ -" timestamp ======= for (const dt of dateTypes) { @@ -218,7 +246,41 @@ function registerDatetimeOperators(schema: _ISchema) { +/** jsonb key existence: a top-level object key, a string element of a top-level array, or the string scalar itself */ +function jsonbHasKey(doc: any, key: string): boolean { + if (Array.isArray(doc)) { + return doc.some(x => x === key); + } + if (doc && typeof doc === 'object') { + return Object.prototype.hasOwnProperty.call(doc, key); + } + return doc === key; +} + function registerJsonOperators(schema: _ISchema) { + // ======= "jsonb ? text" / "?|" (any) / "?&" (all) key existence + schema.registerOperator({ + operator: '?', + left: Types.jsonb, + right: Types.text(), + returns: Types.bool, + implementation: (a, b: string) => jsonbHasKey(a, b), + }); + schema.registerOperator({ + operator: '?|', + left: Types.jsonb, + right: Types.text().asArray(), + returns: Types.bool, + implementation: (a, b: string[]) => b.some(k => k !== null && jsonbHasKey(a, k)), + }); + schema.registerOperator({ + operator: '?&', + left: Types.jsonb, + right: Types.text().asArray(), + returns: Types.bool, + implementation: (a, b: string[]) => b.every(k => k !== null && jsonbHasKey(a, k)), + }); + // ======= "json @> json" query operator schema.registerOperator({ operator: '@>', diff --git a/src/schema/pg-catalog/pg-constraints-list.ts b/src/schema/pg-catalog/pg-constraints-list.ts index aa00ec78..94c5c6cc 100644 --- a/src/schema/pg-catalog/pg-constraints-list.ts +++ b/src/schema/pg-catalog/pg-constraints-list.ts @@ -81,7 +81,7 @@ export class PgConstraintTable extends ReadOnlyTable implements _ITable { } getIndex(forValue: IValue): _IIndex | nil { - if (forValue.id === 'table_name') { + if (forValue?.id === 'table_name') { return new TableIndex(this, forValue); } return null; diff --git a/src/schema/prepared.ts b/src/schema/prepared.ts index 3a9e4111..5b4a17b6 100644 --- a/src/schema/prepared.ts +++ b/src/schema/prepared.ts @@ -27,6 +27,8 @@ function isSchemaChange(s: Statement): boolean { case 'drop index': case 'drop sequence': case 'drop table': + case 'drop view': + case 'drop materialized view': case 'drop trigger': case 'drop type': return true; @@ -40,6 +42,66 @@ function isSchemaChange(s: Statement): boolean { } } +/** + * Where a top-level query call starts: the transaction block a previous call left open + * (BEGIN without COMMIT/ROLLBACK yet), or a fresh implicit transaction forked from root. + * + * Inside a block that a failed statement aborted, Postgres refuses everything but the + * statement that ends the block; COMMIT then rolls back, like it does in Postgres. + */ +function startCall(db: _IDb, statements: Statement[]): _Transaction { + const open = db.sessionTx; + db.sessionTx = null; + if (!open) { + return db.data.fork(); + } + if (open.aborted) { + const first = statements[0]?.type; + if (first === 'rollback' && (statements[0] as any).to) { + // ROLLBACK TO SAVEPOINT is how a client recovers an aborted block without ending it + // (what drivers' nested transactions do after an error): rewind to the savepoint and + // carry on in the block. If the savepoint does not exist, that statement fails and the + // block stays aborted. + open.aborted = false; + return open; + } + if (first === 'rollback' || first === 'commit') { + // the block is over either way (a COMMIT on an aborted block is a ROLLBACK in + // postgres): undo it, schema included, and carry on in whatever enclosed it + statements.splice(0, 1); + const after = open.explicitBlock?.rollback() ?? db.data; + return after.isChild ? after : db.data.fork(); + } + db.sessionTx = open; + throw new QueryError('current transaction is aborted, commands ignored until end of transaction block', '25P02'); + } + return open; +} + +/** End of a top-level call: commit, unless a BEGIN block is still open, in which case keep it for the next call. */ +function endCall(db: _IDb, state: _Transaction) { + if (state.inExplicitBlock) { + db.sessionTx = state; + return; + } + // run any deferred (DEFERRABLE INITIALLY DEFERRED) constraint checks before + // finalizing - a violation aborts the batch, so nothing persists to root + runDeferredChecks(state); + state.fullCommit(); +} + +/** A statement failed: if it ran inside a BEGIN block, the block is now aborted until ROLLBACK. */ +function failCall(db: _IDb, state: _Transaction) { + if (state.inExplicitBlock) { + state.aborted = true; + db.sessionTx = state; + return; + } + // the call's implicit transaction is dropped, so its data is gone: put the schema back too + // (a failed migration must not leave half its tables behind) + state.discardAll(); +} + let _paramList: Parameter[] | null = null; const paramsVisitor = astVisitor(() => ({ // a PREPAREd statement's own $-parameters are bound at EXECUTE time; they are not @@ -122,36 +184,48 @@ class PreparedQueryNoDescribe implements _IPreparedQuery { } bind(args: any[]): _IBoundQuery { - // Start an implicit transaction - let t = this.schema.db.data.fork(); + const db = this.schema.db; + const query = [...this.query]; + let t = startCall(db, query); const results: StatementResult[] = []; - for (const stmt of this.query) { - const s = new StatementExec(this.schema, stmt, this.singleSql); - - const compiled = s.compile(); - - // store last select for debug purposes - if (compiled instanceof SelectExec) { - this.schema.lastSelect = compiled.selection; + try { + for (const stmt of query) { + const s = new StatementExec(this.schema, stmt, this.singleSql); + + const compiled = s.compile(); + + // store last select for debug purposes + if (compiled instanceof SelectExec) { + this.schema.lastSelect = compiled.selection; + } + + // execute statement + const r = s.executeStatement(t, args); + r.result.rows = cleanResults(r.result.rows); + results.push(r); + t = r.state; } - - // execute statement - const r = s.executeStatement(t, args); - r.result.rows = cleanResults(r.result.rows); - results.push(r); - t = r.state; + } catch (e) { + failCall(db, t); + throw e; + } + if (!results.length) { + // the call was only the ROLLBACK/COMMIT closing an aborted block + endCall(db, t); + return new NoDescribeBound(this, [{ result: { command: 'ROLLBACK', fields: [], rowCount: 0, rows: [], location: { start: 0, end: 0 } } as any, state: t }], true); } return new NoDescribeBound(this, results); } } class NoDescribeBound implements _IBoundQuery { - constructor(private parent: PreparedQueryNoDescribe, private results: StatementResult[]) { + constructor(private parent: PreparedQueryNoDescribe, private results: StatementResult[], private ended = false) { } executeAll(): QueryResult { const last = this.results[this.results.length - 1]!; - runDeferredChecks(last.state); - last.state.fullCommit(); + if (!this.ended) { + endCall(this.parent.schema.db, last.state); + } this.parent.executed?.(); return last.result; } @@ -242,19 +316,42 @@ class Bound implements _IBoundQuery { } private *doExecute(outerTx?: _Transaction): IterableIterator { - // Start an implicit transaction - // (to avoid messing global data if an operation fails mid-write) - let t = outerTx ?? this.parent.schema.db.data.fork(); - + // Start an implicit transaction (to avoid messing global data if an operation fails + // mid-write), or resume the BEGIN block a previous call left open + const db = this.parent.schema.db; + let stmts = this.stmts; + let t: _Transaction; + if (outerTx) { + t = outerTx; + } else { + const asts = stmts.map(s => s.statement); + t = startCall(db, asts); + if (asts.length < stmts.length) { + // the leading ROLLBACK/COMMIT closed an aborted block + stmts = stmts.slice(stmts.length - asts.length); + if (!stmts.length) { + yield { result: { command: 'ROLLBACK', fields: [], rowCount: 0, rows: [], location: { start: 0, end: 0 } } as any, state: t }; + endCall(db, t); + return; + } + } + } let lastResult: StatementResult; - for (const s of this.stmts) { - // Execute statement - const r = s.executeStatement(t, this.args); - r.result.rows = cleanResults(r.result.rows); - yield r; - lastResult = r; - t = r.state; + try { + for (const s of stmts) { + // Execute statement + const r = s.executeStatement(t, this.args); + r.result.rows = cleanResults(r.result.rows); + yield r; + lastResult = r; + t = r.state; + } + } catch (e) { + if (!outerTx) { + failCall(db, t); + } + throw e; } if (this.parent.lastSelect) { @@ -263,10 +360,7 @@ class Bound implements _IBoundQuery { this.parent.executed?.(); if (!outerTx) { - // run any deferred (DEFERRABLE INITIALLY DEFERRED) constraint checks before - // finalizing - a violation aborts the batch, so nothing persists to root - runDeferredChecks(lastResult!.state); - lastResult!.state.fullCommit(); + endCall(db, lastResult!.state); } } diff --git a/src/schema/schema.ts b/src/schema/schema.ts index 7f5ae16b..56673e8c 100644 --- a/src/schema/schema.ts +++ b/src/schema/schema.ts @@ -1,6 +1,6 @@ import { ISchema, DataType, IType, RelationNotFound, Schema, QueryResult, SchemaField, nil, FunctionDefinition, PermissionDeniedError, TypeNotFound, ArgDefDetails, IEquivalentType, QueryInterceptor, ISubscription, QueryError, typeDefToStr, OperatorDefinition, QueryOrAst, IPreparedQuery } from '../interfaces'; import { _IDb, _ISelection, _ISchema, _Transaction, _ITable, _SelectExplanation, _Explainer, IValue, _IIndex, _IType, _IRelation, QueryObjOpts, _ISequence, _INamedIndex, RegClass, Reg, TypeQuery, asType, _ArgDefDetails, BeingCreated, _FunctionDefinition, _OperatorDefinition } from '../interfaces-private'; -import { asSingleQName, ignore, isType, notNil, parseRegClass, randomString, schemaOf } from '../utils'; +import { asSingleQName, ignore, isType, notNil, parseRegClass, randomString, schemaOf, hasExecutionCtx } from '../utils'; import { typeSynonyms, Types } from '../datatypes'; import { DropFunctionStatement, BinaryOperator, QName, DataTypeDef, CreateSequenceOptions, CreateExtensionStatement, Statement } from 'pgsql-ast-parser'; import { MemoryTable } from '../table'; @@ -100,6 +100,12 @@ export class DbSchema implements _ISchema, ISchema { return ret; } catch (e) { this.db.raiseGlobal('query-failed', query); + // a top-level statement that fails to compile (unknown column, uuid = text, ...) inside a + // BEGIN block aborts it, like one failing at run time: COMMIT then rolls back + const open = this.db.sessionTx; + if (open?.inExplicitBlock && !hasExecutionCtx()) { + open.aborted = true; + } throw e; } } @@ -393,10 +399,14 @@ export class DbSchema implements _ISchema, ISchema { - declareTable(table: Schema, noSchemaChange?: boolean): MemoryTable { - const trans = this.db.data.fork(); + declareTable(table: Schema, noSchemaChange?: boolean, inTransaction?: _Transaction): MemoryTable { + // CREATE TABLE builds the table inside the statement's transaction (DDL is transactional); + // the JS API (db.public.declareTable) has none, so it commits on its own + const trans = inTransaction ?? this.db.data.fork(); const ret = new MemoryTable(this, trans, table).register(); - trans.commit(); + if (!inTransaction) { + trans.commit(); + } if (!noSchemaChange) { this.db.onSchemaChange(); } diff --git a/src/schema/view.ts b/src/schema/view.ts index 5d69fc91..c84f8eaa 100644 --- a/src/schema/view.ts +++ b/src/schema/view.ts @@ -65,6 +65,7 @@ export class View extends FilterBase implements _IView { } drop(t: _Transaction): void { - throw new Error('Method not implemented.'); + this.ownerSchema._reg_unregister(this); + this.db.onSchemaChange(); } } \ No newline at end of file diff --git a/src/table.ts b/src/table.ts index dd309b0b..f5efcb2f 100644 --- a/src/table.ts +++ b/src/table.ts @@ -72,6 +72,7 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I throw new QueryError(`policy "${policy.name}" for table "${this.name}" already exists`, '42710'); } this.rls.policies.push(policy); + this.db.onSchemaChange(); } dropPolicy(name: string, ifExists: boolean): void { @@ -83,6 +84,7 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I throw new QueryError(`policy "${name}" for table "${this.name}" does not exist`, '42704'); } this.rls.policies.splice(idx, 1); + this.db.onSchemaChange(); } setRowLevelSecurity(action: 'enable' | 'disable' | 'force' | 'no force'): void { @@ -92,6 +94,7 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I case 'force': this.rls.forced = true; break; case 'no force': this.rls.forced = false; break; } + this.db.onSchemaChange(); } readonly triggers: TableTriggers = emptyTriggers(); @@ -239,6 +242,8 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I type: this.ownerSchema.getType(column.dataType), }; delete (tp as any as Optional).dataType; + // keep the type as written, for information_schema (see ColRef.declaredType) + (tp as any).declaredType = column.dataType; return this.addColumn(tp, t); } @@ -439,7 +444,8 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I const key = index.buildKey(toInsert, t); const found = index.eqFirst(key, t); if (found) { - return found; // ignore. + // skipped, like the untargeted form: RETURNING yields nothing for it + return null; } } } else { @@ -635,10 +641,28 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I ?? (this.name + '_constraint_' + (++this.cstGen)); } + /** + * Postgres' name for an unnamed constraint/index: __…_, then + * 1, 2… until free. Migrations rely on it: `alter table t drop constraint if + * exists t_status_check` before re-adding a widened check is how enums get extended. + */ + autoRelName(columns: string[], suffix: string, taken: (name: string) => boolean): string { + const base = [this.name, ...columns, suffix].join('_'); + let name = base; + for (let i = 1; taken(name); i++) { + name = base + i; + } + return name; + } + addCheck(_t: _Transaction, check: Expr, constraintName?: string): _IConstraint { - constraintName = this.constraintNameGen(constraintName); - this.checkNoConstraint(constraintName); const getter = withSelection(this.selection, () => buildValue(check).cast(Types.bool)); + if (!constraintName) { + // one referenced column:
__check; several:
_check + const cols = [...new Set([...getter.usedColumns].map(c => c.id!))]; + constraintName = this.autoRelName(cols.length === 1 ? cols : [], 'check', n => this.constraintsByName.has(n) || !!this.ownerSchema.getOwnObject(n)); + } + this.checkNoConstraint(constraintName); const checkVal = (t: _Transaction, v: any) => { const value = getter.get(v, t); @@ -682,6 +706,10 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I value: getter, }); } + if (!_indexName && _type === 'unique') { + // a UNIQUE constraint (not a CREATE UNIQUE INDEX) is
__key + _indexName = this.autoRelName(expressions.map(e => e.name), 'key', n => this.constraintsByName.has(n) || !!this.ownerSchema.getOwnObject(n)); + } return this.createIndex(t, { columns: keys, primary: _type === 'primary', @@ -706,7 +734,9 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I const ihash = indexHash(expressions.columns.map(x => x.value)); - const indexName = this.determineIndexRelName(expressions.indexName, ihash, expressions.ifNotExists, 'idx'); + // an unnamed index is
__idx; an expression contributes its function name (lower(x) -> lower), else "expr" + const nameParts = expressions.columns.map(c => c.value.id ?? 'expr'); + const indexName = this.determineIndexRelName(expressions.indexName, nameParts, expressions.ifNotExists, 'idx'); if (!indexName) { return null; } @@ -747,7 +777,7 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I return ret; } - private determineIndexRelName(indexName: string | nil, ihash: string, ifNotExists: boolean | nil, sufix: string): string | nil { + private determineIndexRelName(indexName: string | nil, nameParts: string[], ifNotExists: boolean | nil, sufix: string): string | nil { if (indexName) { if (this.ownerSchema.getOwnObject(indexName)) { if (ifNotExists) { @@ -757,12 +787,7 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I } return indexName; } else { - const baseName = indexName = `${this.name}_${ihash}_${sufix}`; - let i = 1; - while (this.ownerSchema.getOwnObject(indexName)) { - indexName = baseName + (i++); - } - return indexName!; + return this.autoRelName(nameParts, sufix, n => !!this.ownerSchema.getOwnObject(n) || this.constraintsByName.has(n)); } } @@ -860,7 +885,7 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I const ihash = indexHash(cst.localColumns.map(x => x.name)); let constraintName: string | nil; if (cst.constraintName?.name) { - constraintName = this.determineIndexRelName(cst.constraintName.name, ihash, false, 'fk'); + constraintName = this.determineIndexRelName(cst.constraintName.name, [], false, 'fk'); } else { // Postgres convention for an unnamed FK:
__..._fkey // (a numeric suffix is appended on collision, like Postgres) @@ -876,9 +901,35 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I } const ret = new ForeignKey(constraintName) .install(t, cst, this); + const name = constraintName; + ret.onUninstalled = () => this.constraintsByName.delete(name); return new ConstraintWrapper(this.constraintsByName, ret); } + /** + * Foreign keys on any table, this one included, that reference this table - narrowed to the + * ones through `viaColumns` when given. Postgres refuses to DROP the table or a referenced + * column out from under them ("other objects depend on it") unless CASCADE. + */ + referencingForeignKeys(viaColumns?: string[]): ForeignKey[] { + const out: ForeignKey[] = []; + for (const schema of this.ownerSchema.db.listSchemas()) { + for (const tbl of schema.listTables()) { + for (const c of tbl.listConstraints?.() ?? []) { + const fk = ((c as any).wrapped ?? c) as ForeignKey; + if (fk.constraintKind !== 'foreign key' || fk.referencedTable !== this) { + continue; + } + if (viaColumns && !fk.foreignColumns.some(fc => viaColumns.includes(fc))) { + continue; + } + out.push(fk); + } + } + } + return out; + } + getConstraint(constraint: string): _IConstraint | nil { return this.constraintsByName.get(constraint); } @@ -965,6 +1016,8 @@ export class MemoryTable extends DataSourceBase implements IMemoryTable, _I for (const map of this.indexByHashAndName.values()) { for (const i of map.values()) { i.index.dropFromData(t); + // free the index's name too, or re-creating the table hits "_pkey already exists" + this.ownerSchema._reg_unregister(i.index); } } // todo should also check foreign keys, cascade, ... diff --git a/src/tests/conversions.spec.ts b/src/tests/conversions.spec.ts index 68edd03c..da45a763 100644 --- a/src/tests/conversions.spec.ts +++ b/src/tests/conversions.spec.ts @@ -32,7 +32,8 @@ describe('Conversions', () => { db.public.none(`create table test(value decimal)`); db.public.none(`insert into test(value) values ('42.5')`); const many = db.public.many(`select value from test where value is not null`); - expect(many).toEqual([{ value: 42.5 }]); + // numeric is a decimal string whichever way it was inserted (as node-postgres returns it) + expect(many).toEqual([{ value: '42.5' }]); }); it('incompatible decimal with string', () => { diff --git a/src/tests/corpus-parity.spec.ts b/src/tests/corpus-parity.spec.ts new file mode 100644 index 00000000..3068b01b --- /dev/null +++ b/src/tests/corpus-parity.spec.ts @@ -0,0 +1,604 @@ +import { describe, it, beforeEach, expect } from 'bun:test'; + +import { newDb } from '../db'; +import { IMemoryDb } from '../interfaces'; +import { expectQueryError } from './test-utils'; + +// Gaps found by running real project migrations through pg-mem and PGlite side by side +// (tools/corpus-diff). Each case is distilled from a statement Postgres and pg-mem disagreed on. + +describe('corpus parity', () => { + + let db: IMemoryDb; + let many: (str: string) => any[]; + let none: (str: string) => void; + beforeEach(() => { + db = newDb(); + many = db.public.many.bind(db.public); + none = db.public.none.bind(db.public); + }); + + describe('a transaction block spans query calls', () => { + // Drivers send BEGIN, the work and ROLLBACK as separate round-trips. pg-mem used to + // commit at the end of every call, so the rollback undid nothing. + beforeEach(() => none(`create table k (id int primary key, n int); insert into k values (1, 1), (2, 2)`)); + const ns = () => many(`select n from k order by id`).map(r => r.n); + + it('ROLLBACK undoes work sent in earlier calls', () => { + none(`begin`); + none(`delete from k`); + expect(many(`select count(*) from k`)).toEqual([{ count: 0 }]); + none(`rollback`); + expect(ns()).toEqual([1, 2]); + }); + + it('COMMIT keeps it', () => { + none(`begin`); + none(`update k set n = 9`); + none(`commit`); + expect(ns()).toEqual([9, 9]); + }); + + it('a failed statement aborts the block until it ends, and COMMIT then rolls back', () => { + none(`begin`); + none(`update k set n = 5`); + expectQueryError(() => none(`insert into k values (1, 0)`), /duplicate key/); + expectQueryError(() => none(`select 1`), /current transaction is aborted/); + none(`commit`); + expect(ns()).toEqual([1, 2]); + expect(many(`select 1 as x`)).toEqual([{ x: 1 }]); + }); + + it('a statement that fails to compile aborts the block too', () => { + none(`begin`); + none(`insert into k values (3, 3)`); + expectQueryError(() => many(`select * from k where nope = 1`), /nope/); + expectQueryError(() => many(`select 1`), /current transaction is aborted/); + none(`commit`); + expect(ns()).toEqual([1, 2]); + }); + + it('ROLLBACK TO SAVEPOINT recovers an aborted block without ending it', () => { + // the agent validator's attempt(): savepoint, run the model's SQL, rewind on error + none(`begin`); + none(`create table keep (id int); insert into k values (3, 3)`); + for (const bad of [`create table t1 (id uuid); select * from t1 where id = 'x'::text`, `create table t2 (id int); insert into t2 values (1/0)`]) { + none(`savepoint attempt`); + expectQueryError(() => none(bad)); + none(`rollback to savepoint attempt`); + none(`release savepoint attempt`); + } + expectQueryError(() => none(`insert into k values (4, 1/0)`)); + expectQueryError(() => none(`rollback to savepoint nope`), /savepoint "nope" does not exist/); + expectQueryError(() => many(`select 1`), /current transaction is aborted/); + none(`rollback`); + none(`begin; savepoint s; insert into k values (5, 5)`); + expectQueryError(() => none(`insert into k values (6, 1/0)`)); + none(`rollback to savepoint s; insert into k values (7, 7); commit`); + expect(ns()).toEqual([1, 2, 7]); + expect(many(`select table_name from information_schema.tables where table_name in ('keep', 't1', 't2')`)).toEqual([]); + }); + + it('a re-declared savepoint is the newest one', () => { + none(`begin; savepoint s; savepoint t; savepoint s; rollback to savepoint s; release savepoint t; rollback`); + }); + + it('savepoints that see no DDL do not copy the schema', () => { + none(`begin`); + for (let i = 0; i < 50; i++) { + none(`savepoint s${i}; create table x${i} (id int); rollback to savepoint s${i}; release savepoint s${i}`); + } + none(`savepoint a; create table kept (id int); savepoint b; create table gone (id int); rollback to savepoint b; commit`); + expect(many(`select table_name from information_schema.tables where table_name in ('kept', 'gone', 'x0')`)).toEqual([{ table_name: 'kept' }]); + }); + + it('savepoints work across calls', () => { + none(`begin`); + none(`insert into k values (3, 3)`); + none(`savepoint a`); + none(`insert into k values (4, 4)`); + none(`rollback to savepoint a`); + none(`commit`); + expect(ns()).toEqual([1, 2, 3]); + }); + + it('statements outside a block still autocommit', () => { + none(`update k set n = 7 where id = 1`); + none(`rollback`); + expect(ns()).toEqual([7, 2]); + }); + }); + + describe('typed values do not cross type categories implicitly', () => { + // pg-mem accepted `auth.uid() = `; postgres has no uuid = text operator and + // refuses at CREATE POLICY - a shipped migration failed exactly there. + beforeEach(() => none(`create table t (u uuid, s text, i int, bi bigint, b bool, ts timestamptz, d date, j jsonb, v varchar(10), n numeric)`)); + const rejects = (expr: string) => expectQueryError(() => many(`select 1 from t where ${expr}`)); + const accepts = (expr: string) => expect(() => many(`select 1 from t where ${expr}`)).not.toThrow(); + + it('rejects comparisons postgres has no operator for', () => { + for (const e of ['u = s', 's = u', 'b = s', 's = true', 'v = u', 'u in (s)', 'u is not distinct from s', + 'coalesce(u, s) is null', `s in ('a', 1)`, 'lower(u) = s', 's like 1', `i like 'x'`, + 'u in (select s from t)', 'u not in (select s from t)', 's in (select u from t)']) { + rejects(e); + } + }); + + it('still coerces untyped literals and widens within a category', () => { + for (const e of [`u = '00000000-0000-0000-0000-000000000001'`, `i = '1'`, 'i = bi', 'bi = i', 'i = n', 'd = ts', 'ts > d', + 'd = now()', 's = v', 'u::text = s', `s || i = 'x1'`, `j ? s`, `concat(u, 'x') = s`]) { + accepts(e); + } + }); + + it('binds parameters as untyped', () => { + none(`insert into t (u) values ('00000000-0000-0000-0000-000000000001')`); + expect(db.public.prepare('select count(*) as c from t where u = $1').bind(['00000000-0000-0000-0000-000000000001']).executeAll().rows) + .toEqual([{ c: 1 }]); + }); + + it('has jsonb key-existence operators', () => { + expect(many(`select '{"a":1}'::jsonb ? 'a' as x, '["a"]'::jsonb ? 'a' as y, '{"a":1}'::jsonb ?| array['z','a'] as z, '{"a":1}'::jsonb ?& array['z','a'] as w`)) + .toEqual([{ x: true, y: true, z: true, w: false }]); + }); + }); + + describe('CREATE POLICY binds its predicates', () => { + beforeEach(() => none(`create schema auth; + create function auth.uid() returns uuid as $$ select null::uuid $$ language sql stable; + create table p (id text primary key, owner uuid, title text)`)); + + it('rejects an operator that does not exist for the operand types', () => { + expectQueryError(() => none(`create policy x on p for select using (auth.uid() = id)`), /cannot cast|operator does not exist/); + }); + it('rejects an unknown column', () => { + expectQueryError(() => none(`create policy x on p for select using (nope = 1)`), /nope/); + }); + it('rejects a non-boolean predicate', () => { + expectQueryError(() => none(`create policy x on p for select using (title)`), /must be type boolean/); + }); + it('rejects USING on INSERT and WITH CHECK on SELECT/DELETE', () => { + expectQueryError(() => none(`create policy x on p for insert using (auth.uid() = owner)`), /only WITH CHECK/); + expectQueryError(() => none(`create policy x on p for select with check (auth.uid() = owner)`), /WITH CHECK cannot be applied/); + }); + it('accepts a well-typed policy', () => { + none(`create policy x on p for all using (auth.uid() = owner) with check (auth.uid() = owner)`); + }); + }); + + describe('dependency checks on DDL', () => { + beforeEach(() => none(`create table parent (id text primary key, owner text); + create table child (id text primary key, parent_id text references parent(id), note text); + create policy c on child for select using (exists (select 1 from parent p where p.id = child.parent_id and p.owner = 'x'))`)); + + it('refuses to retype a column any policy reads, even from another table', () => { + expectQueryError(() => none(`alter table child alter column parent_id type text`), /used in a policy definition/); + expectQueryError(() => none(`alter table parent alter column owner type varchar(20)`), /used in a policy definition/); + none(`alter table child alter column note type varchar(50)`); + // p.id in the subquery is parent.id, not child.id + none(`alter table child alter column id type varchar(50)`); + }); + it('refuses to drop a column a foreign key or a policy depends on, unless CASCADE', () => { + expectQueryError(() => none(`alter table parent drop column id`), /other objects depend on it/); + expectQueryError(() => none(`alter table parent drop column owner`), /other objects depend on it/); + none(`alter table parent drop column owner cascade`); + expect(many(`select policyname from pg_policies where tablename = 'child'`)).toEqual([]); + }); + it('resolves the trigger function at CREATE TRIGGER', () => { + expectQueryError(() => none(`create trigger t before update on child for each row execute function nope()`), /does not exist/); + none(`create function f() returns int language sql as $$ select 1 $$`); + expectQueryError(() => none(`create trigger t before update on child for each row execute function f()`), /must return type trigger/); + }); + }); + + describe('unnamed constraints and indexes get postgres names', () => { + beforeEach(() => none(`create table x (id int primary key, a int check (a > 0), b int, e text, f text, + check (a > b), check (b > 0), unique (e, f), unique (e))`)); + it('names checks
__check, or
_check over several columns', () => { + // dropping by the postgres name is how migrations widen an enum-like check + none(`alter table x drop constraint x_a_check`); + none(`alter table x drop constraint x_check`); + none(`alter table x drop constraint x_b_check`); + none(`alter table x add check (b < 100)`); + none(`alter table x drop constraint x_b_check`); + }); + it('names unique constraints
__key and indexes
__idx', () => { + none(`create index on x (a, b)`); + expect(many(`select indexname from pg_indexes where tablename = 'x' order by 1`).map(r => r.indexname)) + .toEqual(['x_a_b_idx', 'x_e_f_key', 'x_e_key', 'x_pkey']); + }); + }); + + describe('three-valued IN', () => { + it('a NULL operand passes a CHECK (col IN (...))', () => { + none(`create table r (v text check (v in ('a', 'b')))`); + none(`insert into r values (null)`); + expectQueryError(() => none(`insert into r values ('c')`), /r_v_check/); + }); + it('NULL IN / NOT IN a list holding NULL is NULL', () => { + expect(many(`select (null in (1, 2)) is null as a, (1 not in (2, null)) is null as b, 1 in (1, null) as c`)) + .toEqual([{ a: true, b: true, c: true }]); + }); + }); + + describe('CREATE TABLE IF NOT EXISTS on an existing table', () => { + it('is a no-op even when the new definition has constraints', () => { + none(`create table v (id text primary key)`); + none(`create table if not exists v (id text primary key, n int not null default 0 check (n > 0) references v(id))`); + expect(many(`select column_name from information_schema.columns where table_name = 'v'`)).toEqual([{ column_name: 'id' }]); + }); + }); + + describe('numeric input', () => { + beforeEach(() => none(`create table n (i int, s smallint, b bigint, d numeric(10, 2), f float)`)); + it('range-checks integers', () => { + expectQueryError(() => none(`insert into n (i) values (3000000000)`), /integer out of range/); + expectQueryError(() => none(`insert into n (i) values ('3000000000')`), /integer out of range/); + expectQueryError(() => none(`insert into n (s) values (70000)`), /smallint out of range/); + none(`insert into n (i, s, b) values (2147483647, 32767, 3000000000)`); + }); + it('parses text strictly into the stored representation', () => { + expectQueryError(() => none(`insert into n (i) values ('12abc')`), /invalid input syntax/); + none(`insert into n (d, f) values ('5200.456', '1.5')`); + expect(many(`select d, d + 1 as d1, f from n where d is not null`)).toEqual([{ d: '5200.46', d1: '5201.46', f: 1.5 }]); + }); + }); + + describe('transaction time', () => { + it('now() and current_timestamp are constant within a transaction', () => { + none(`begin`); + const [{ a }] = many(`select now() as a`); + const t0 = Date.now(); while (Date.now() - t0 < 5) { /* let the clock move */ } + const [{ b, c }] = many(`select now() as b, current_timestamp as c`); + none(`commit`); + expect(b.getTime()).toBe(a.getTime()); + expect(c.getTime()).toBe(a.getTime()); + }); + it('current_date is a date (midnight), usable in a column default', () => { + none(`create table d (x date not null default current_date, y date)`); + none(`insert into d (y) values (current_date + 2)`); + const [{ x, y }] = many(`select x, y from d`); + expect(x.getUTCHours()).toBe(0); + expect((y.getTime() - x.getTime()) / 86400_000).toBe(2); + }); + }); + + describe('information_schema.columns reports postgres type names', () => { + it('matches data_type / udt_name', () => { + none(`create table c (a varchar(20), b smallint, c real, d numeric, e boolean, f timestamptz, g text[], h char(3))`); + expect(many(`select column_name, data_type, udt_name from information_schema.columns where table_name = 'c' order by ordinal_position`) + .map(r => `${r.column_name}:${r.data_type}:${r.udt_name}`)) + .toEqual(['a:character varying:varchar', 'b:smallint:int2', 'c:real:float4', 'd:numeric:numeric', 'e:boolean:bool', + 'f:timestamp with time zone:timestamptz', 'g:ARRAY:_text', 'h:character:bpchar']); + }); + }); + + describe('supabase function & view DDL', () => { + it('accepts SECURITY DEFINER ... SET search_path, and DROP VIEW / DROP FUNCTION f()', () => { + none(`create table t (id int)`); + none(`create or replace function public.h() returns trigger language plpgsql security definer set search_path = public as $$ begin return new; end $$`); + none(`create function g() returns int language sql stable set search_path = '' cost 5 parallel safe as $$ select 1 $$`); + none(`create view v with (security_invoker = true) as select * from t`); + none(`drop view if exists v; drop view if exists v; drop function if exists g() cascade`); + }); + }); + + describe('plpgsql', () => { + beforeEach(() => none(`create table markets (id text primary key, title text, status text, meta jsonb); + insert into markets values ('a', 'Rain?', 'open', '{"k": "v"}'), ('b', 'Snow?', 'settled', null)`)); + + it('reads multi-character operators (->>) inside bodies', () => { + none(`create function k(mid text) returns text language plpgsql as $$ begin return (select meta->>'k' from markets where id = mid); end $$`); + expect(many(`select k('a') as x`)).toEqual([{ x: 'v' }]); + }); + + it('supports %ROWTYPE and %TYPE variables, resolved at first call', () => { + none(`create function st(mid text) returns text language plpgsql as $$ + declare m public.markets%rowtype; s markets.status%type; + begin select * into m from markets where id = mid; s := m.status; + if m.status <> 'open' then return 'closed:' || m.title; end if; return 'open:' || s; end $$`); + expect(many(`select st('a') as a, st('b') as b`)).toEqual([{ a: 'open:open', b: 'closed:Snow?' }]); + }); + + it('runs a SECURITY DEFINER body as its owner, so RLS does not filter inside it', () => { + none(`create role authenticated; + create table members (crew int, uid text); alter table members enable row level security; + create policy own on members for select to authenticated using (uid = current_setting('app.uid')); + insert into members values (1, 'me'), (1, 'you'), (2, 'you'); + create function crew_size(c int) returns int language sql stable security definer as $$ select count(*)::int from members where crew = c $$; + create function crew_size_invoker(c int) returns int language sql stable as $$ select count(*)::int from members where crew = c $$`); + none(`select set_config('app.uid', 'me', false)`); + none(`set role authenticated`); + expect(many(`select crew_size(1) as d, crew_size_invoker(1) as i, current_user as u`)).toEqual([{ d: 2, i: 1, u: 'authenticated' }]); + }); + }); + + describe('date/time and record arithmetic', () => { + it('subtracts timestamps into an interval, and dates into days', () => { + expect(many(`select (timestamptz '2026-01-01' - timestamptz '2026-01-02 06:30:00')::text as a, + current_date - (current_date - 3) as b`)).toEqual([{ a: '-1 days -06:30:00', b: 3 }]); + }); + it('renders intervals and rows as text like postgres', () => { + expect(many(`select (interval '3 months')::text as a, (interval '0')::text as b`)).toEqual([{ a: '3 mons', b: '00:00:00' }]); + expect(many(`select 'x-' || s as r from (values ('a b')) as s(suffix)`)).toEqual([{ r: 'x-("a b")' }]); + }); + }); + + describe('DROP and re-CREATE', () => { + it('removes foreign keys into a table dropped with CASCADE from the catalogues', () => { + none(`create table pr (id int primary key); create table li (u int references pr(id) on delete cascade, v int references pr(id)); + drop table pr cascade; create table pr (id int primary key)`); + expect(many(`select constraint_name from information_schema.table_constraints where table_name = 'li' and constraint_type = 'FOREIGN KEY'`)).toEqual([]); + }); + + it('frees the dropped table index names', () => { + none(`create table p (id int primary key, e text unique)`); + none(`drop table if exists p cascade`); + none(`create table p (id int primary key, e text unique)`); + }); + }); + + describe('grantees must exist', () => { + it('rejects GRANT / CREATE POLICY ... TO an unknown role', () => { + none(`create table p (id int)`); + expectQueryError(() => none(`grant select on p to service_role`), /role "service_role" does not exist/); + expectQueryError(() => none(`create policy x on p for select to service_role using (true)`), /role "service_role" does not exist/); + expectQueryError(() => none(`grant select on nope to public`), /nope/); + none(`create role service_role; grant all on p to service_role, public`); + }); + }); + + describe('DDL is transactional', () => { + // expectations checked against PGlite (real Postgres) + const n = (q: string) => many(q)[0].n; + + it('a failed multi-statement call leaves nothing behind', () => { + expectQueryError(() => none(`create table a(x int); insert into a values (1); select nope_fn()`)); + expect(n(`select count(*)::int as n from pg_tables where tablename = 'a'`)).toBe(0); + }); + it('ROLLBACK undoes CREATE TABLE, and restores a dropped table with its rows', () => { + none(`begin`); none(`create table b(x int)`); none(`rollback`); + expect(n(`select count(*)::int as n from pg_tables where tablename = 'b'`)).toBe(0); + none(`create table c(x int primary key); insert into c values (1), (2)`); + none(`begin`); none(`drop table c`); none(`rollback`); + expect(n(`select count(*)::int as n from c`)).toBe(2); + }); + it('restores a dropped foreign key, still enforced', () => { + none(`create table p(id int primary key); create table ch(pid int constraint ch_fk references p(id))`); + none(`begin`); none(`alter table ch drop constraint ch_fk`); none(`rollback`); + expectQueryError(() => none(`insert into ch values (99)`), /ch_fk/); + }); + it('undoes added columns, policies, RLS, functions, triggers and type changes', () => { + none(`create table d(x int, y int)`); + none(`create function d_trg() returns trigger language plpgsql as $$ begin new.y = 42; return new; end $$`); + none(`begin`); + none(`alter table d add column z text`); + none(`alter table d enable row level security`); + none(`create policy pd on d using (true)`); + none(`create function f1() returns int language sql as $$ select 1 $$`); + none(`create trigger tg before insert on d for each row execute function d_trg()`); + none(`alter table d alter column x type text`); + none(`rollback`); + expect(n(`select count(*)::int as n from information_schema.columns where table_name = 'd'`)).toBe(2); + expect(n(`select count(*)::int as n from pg_policies where tablename = 'd'`)).toBe(0); + expect(many(`select rowsecurity from pg_tables where tablename = 'd'`)).toEqual([{ rowsecurity: false }]); + expectQueryError(() => many(`select f1()`), /f1/); + none(`insert into d (x) values (1)`); + expect(many(`select x, y from d`)).toEqual([{ x: 1, y: null }]); + }); + it('ROLLBACK TO SAVEPOINT undoes only the DDL after it', () => { + none(`begin`); none(`create table h(x int)`); none(`savepoint s`); + none(`alter table h add column y int`); none(`create index h_x on h(x)`); + none(`rollback to savepoint s`); none(`commit`); + expect(n(`select count(*)::int as n from information_schema.columns where table_name = 'h'`)).toBe(1); + expect(n(`select count(*)::int as n from pg_indexes where tablename = 'h'`)).toBe(0); + none(`create index h_x on h(x)`); + }); + it('keeps DDL from a committed block', () => { + none(`begin`); none(`create table i(x int)`); none(`insert into i values (5)`); none(`commit`); + expect(many(`select x from i`)).toEqual([{ x: 5 }]); + }); + }); + + describe('referential actions run in postgres order (breadth-first)', () => { + // outcomes checked against PGlite + const count = (q: string) => Number(many(q)[0].n); + + it('a direct NO ACTION reference is checked before a grandchild cascade clears it', () => { + none(`create table u (id int primary key); create table pr (id int primary key references u(id) on delete cascade); + create table tx (id int primary key, acct int not null references pr(id) on delete cascade, by int references u(id)); + insert into u values (1); insert into pr values (1); insert into tx values (1, 1, 1)`); + expectQueryError(() => none(`delete from u where id = 1`), /tx_by_fkey/); + expect(count(`select (select count(*) from pr) + (select count(*) from tx) * 10 as n`)).toBe(11); + }); + it('a direct CASCADE queued before a NO ACTION check satisfies it', () => { + none(`create table u (id int primary key); create table a (uid int references u(id) on delete cascade, by int references u(id)); + insert into u values (1); insert into a values (1, 1)`); + none(`delete from u where id = 1`); + expect(count(`select count(*) as n from a`)).toBe(0); + }); + it('a NO ACTION declared before the CASCADE fires first and fails', () => { + none(`create table u (id int primary key); create table a (by int references u(id), uid int references u(id) on delete cascade); + insert into u values (1); insert into a values (1, 1)`); + expectQueryError(() => none(`delete from u where id = 1`), /a_by_fkey/); + }); + it('cascades through chains and self references', () => { + none(`create table t (id int primary key, parent int references t(id) on delete cascade); + insert into t values (1, null), (2, 1), (3, 2), (4, null)`); + none(`delete from t where id = 1`); + expect(count(`select count(*) as n from t`)).toBe(1); + }); + }); + + describe('column_default reads like pg_get_expr', () => { + it('prints literals as written and expressions without extra parens', () => { + none(`create table o (a numeric default 0.30, b double precision default 0.0, d text[] default array['x','y'], + e timestamptz default now() + interval '2 days', g int default -1, r bigint default 3000000000, i date default current_date)`); + expect(many(`select column_name, column_default from information_schema.columns where table_name = 'o' order by ordinal_position`) + .map(r => r.column_default)) + .toEqual(['0.30', '0.0', `ARRAY['x', 'y']`, `(now() + '2 days'::interval)`, `'-1'`, `'3000000000'`, 'CURRENT_DATE']); + }); + }); + + describe('policy expansion', () => { + beforeEach(() => none(`create role authenticated; + create table profiles (id int primary key, role text); alter table profiles enable row level security; + create policy p_sel on profiles for select using (exists (select 1 from profiles p where p.id = 1 and p.role = 'admin')); + create table cats (id int); alter table cats enable row level security; + create policy c_sel on cats for select using (true); + create policy c_upd on cats for update using (exists (select 1 from profiles p where p.id = 1)); + insert into cats values (1)`)); + + it('fails on a policy that (indirectly) queries its own table, for roles RLS applies to', () => { + none(`set role authenticated`); + expectQueryError(() => many(`select * from profiles`), /infinite recursion detected in policy for relation "profiles"/); + expectQueryError(() => none(`update cats set id = id`), /infinite recursion detected in policy for relation "profiles"/); + expect(many(`select count(*)::int as n from cats`)).toEqual([{ n: 1 }]); + none(`reset role`); + expect(many(`select count(*)::int as n from profiles`)).toEqual([{ n: 0 }]); + }); + }); + + describe('a policy subquery sees the other table\'s policies as they are when it runs', () => { + // the check CREATE POLICY runs used to leave its build cached, so enforcement reused a + // subquery compiled under the other table's policies as they were at CREATE POLICY time + beforeEach(() => none(`create role authenticated; + create table members (team_id int, user_id text); create table teams (id int primary key); + insert into teams values (1), (2); insert into members values (1, 'u1'), (2, 'u2'); + alter table teams enable row level security; alter table members enable row level security`)); + const teams = () => { + none(`set role authenticated`); + try { + return many(`select id from teams order by id`).map(r => r.id); + } finally { + none(`reset role`); + } + }; + + it('a policy created after the one that reads it applies', () => { + none(`create policy t on teams for select using (exists (select 1 from members m where m.team_id = teams.id)); + create policy m on members for select using (user_id = 'u1')`); + expect(teams()).toEqual([1]); + }); + + it('narrowing the other table\'s policy narrows the result', () => { + none(`create policy m on members for select using (true); + create policy t on teams for select using (exists (select 1 from members m where m.team_id = teams.id))`); + expect(teams()).toEqual([1, 2]); + none(`drop policy m on members; create policy m on members for select using (user_id = 'u1')`); + expect(teams()).toEqual([1]); + none(`alter table members disable row level security`); + expect(teams()).toEqual([1, 2]); + }); + + it('replacing a recursive policy clears the recursion error', () => { + none(`create policy t on teams for select using (exists (select 1 from teams x where x.id = teams.id))`); + expectQueryError(() => teams(), /infinite recursion detected in policy for relation "teams"/); + none(`drop policy t on teams; create policy t on teams for select using (id = 1)`); + expect(teams()).toEqual([1]); + }); + + it('adding a recursive policy after a clean query reports the recursion', () => { + none(`create policy t on teams for select using (id = 1)`); + expect(teams()).toEqual([1]); + none(`create policy t2 on teams for select using (exists (select 1 from teams x where x.id = teams.id))`); + expectQueryError(() => teams(), /infinite recursion detected in policy for relation "teams"/); + }); + }); + + describe('RLS on INSERT ... ON CONFLICT', () => { + // checked against PGlite: the conflicting row is subject to the UPDATE policies, and + // DO NOTHING never hands back a row the insert did not write + beforeEach(() => none(`create role alice; + create table docs (id int primary key, owner text, body text); + insert into docs values (1, 'alice', 'a1'), (2, 'bob', 'b2'); + alter table docs enable row level security; + create policy s on docs for select using (owner = current_user); + create policy i on docs for insert with check (owner = current_user); + create policy u on docs for update using (owner = current_user) with check (owner = current_user); + set role alice`)); + const rows = () => { + none(`reset role`); + return many(`select id, owner, body from docs order by id`); + }; + + it('DO UPDATE on a row the role may not update fails', () => { + expectQueryError(() => many(`insert into docs values (2, 'alice', 'x') on conflict (id) do update set body = 'upserted' returning *`), + /violates row-level security policy \(USING expression\)/); + expect(rows()[1]).toEqual({ id: 2, owner: 'bob', body: 'b2' }); + }); + + it('DO UPDATE checks the updated row against WITH CHECK, leaving it untouched on failure', () => { + expect(many(`insert into docs values (1, 'alice', 'x') on conflict (id) do update set body = 'mine' returning body`)).toEqual([{ body: 'mine' }]); + expectQueryError(() => none(`insert into docs values (1, 'alice', 'x') on conflict (id) do update set owner = 'bob'`), + /violates row-level security policy/); + expect(rows()[0]).toEqual({ id: 1, owner: 'alice', body: 'mine' }); + }); + + it('DO NOTHING returns nothing for a conflicting row', () => { + expect(many(`insert into docs values (2, 'alice', 'x') on conflict (id) do nothing returning *`)).toEqual([]); + expect(many(`insert into docs values (2, 'alice', 'x') on conflict do nothing returning *`)).toEqual([]); + }); + }); + + describe('CREATE OR REPLACE TRIGGER', () => { + it('replaces an existing trigger', () => { + none(`create table o (id int, n int); + create function f() returns trigger language plpgsql as $$ begin new.n = 1; return new; end $$; + create function g() returns trigger language plpgsql as $$ begin new.n = 2; return new; end $$; + create or replace trigger t before insert on o for each row execute function f(); + create or replace trigger t before insert on o for each row execute function g(); + insert into o (id) values (1)`); + expect(many(`select n from o`)).toEqual([{ n: 2 }]); + }); + }); + + describe('RLS on index lookups', () => { + it('applies policies to WHERE = … reads and writes', () => { + none(`create role authenticated; create table pr (id int primary key, n text); alter table pr enable row level security; + create policy s on pr for select using (id = 1); create policy u on pr for update using (id = 1); create policy d on pr for delete using (id = 1); + insert into pr values (1, 'a'), (2, 'b')`); + none(`set role authenticated`); + expect(many(`select * from pr where id = 2`)).toEqual([]); + expect(many(`select n from pr where id = 1`)).toEqual([{ n: 'a' }]); + expect(db.public.query(`update pr set n = 'z' where id = 2`).rowCount).toBe(0); + expect(db.public.query(`delete from pr where id in (2)`).rowCount).toBe(0); + none(`reset role`); + expect(many(`select n from pr order by id`)).toEqual([{ n: 'a' }, { n: 'b' }]); + }); + }); + + describe('policy recursion follows postgres rules', () => { + // checked against PGlite + beforeEach(() => none(`create role authenticated; create table pr (id int primary key, n text); alter table pr enable row level security`)); + it('an UPDATE policy reading its own table is fine when that table has no recursive SELECT policy', () => { + none(`create policy s on pr for select using (true); + create policy u on pr for update using (id = 1 or exists (select 1 from pr p where p.id = 2)); set role authenticated`); + none(`update pr set n = n where id = 1`); + }); + it('an UPDATE/DELETE that reads columns expands the recursive SELECT policy; one that does not, does not', () => { + none(`create policy s on pr for select using (id = 1 or exists (select 1 from pr p where p.id = 2)); + create policy u on pr for update using (id = 1); create policy d on pr for delete using (id = 1); set role authenticated`); + expectQueryError(() => none(`update pr set n = 'x' where id = 1`), /infinite recursion/); + expectQueryError(() => none(`delete from pr where id = 1`), /infinite recursion/); + none(`update pr set n = 'x'`); + none(`delete from pr`); + }); + }); + + describe('CREATE OR REPLACE FUNCTION', () => { + it('replaces a trigger function that triggers already use, in a later call', () => { + none(`create table p (id int, updated_at timestamptz); + create or replace function set_updated_at() returns trigger as $$ begin new.updated_at = now(); return new; end $$ language plpgsql; + create trigger p_upd before update on p for each row execute function set_updated_at()`); + none(`create or replace function set_updated_at() returns trigger as $$ begin new.updated_at = '2000-01-01'; return new; end $$ language plpgsql`); + none(`insert into p (id) values (1); update p set id = 2`); + expect(many(`select extract(year from updated_at) as y from p`)).toEqual([{ y: 2000 }]); + }); + it('still refuses to change a return type', () => { + none(`create function f2() returns int language sql as $$ select 1 $$`); + expectQueryError(() => none(`create or replace function f2() returns text language sql as $$ select 'a' $$`), /cannot change return type/); + }); + }); +}); + diff --git a/src/tests/insert.queries.spec.ts b/src/tests/insert.queries.spec.ts index 57b2ea8e..a33844bb 100644 --- a/src/tests/insert.queries.spec.ts +++ b/src/tests/insert.queries.spec.ts @@ -229,8 +229,10 @@ describe('Inserts', () => { }); it('ensures serials are transactional', () => { - expect(many(`create table test(id serial primary key, val text); - insert into test(val) values ('x'); + // the table is created in its own call: in one multi-statement call, ROLLBACK would undo + // the CREATE TABLE too (DDL is transactional, as in postgres) + none(`create table test(id serial primary key, val text)`); + expect(many(`insert into test(val) values ('x'); insert into test(val) values ('x'); insert into test(val) values ('x'); rollback; @@ -331,7 +333,8 @@ describe('Inserts', () => { it('should allow string for bigint columns on insert', () => { none(`create table test(a bigint, b int8);`); - expect(many(`insert into test values ('123456','111') returning a`)).toEqual([{ a: 123456 }]); + // bigint is a digit string whichever way it was inserted (as node-postgres returns it) + expect(many(`insert into test values ('123456','111') returning a`)).toEqual([{ a: '123456' }]); }) it('checks that insert values has enough columns', () => { diff --git a/src/tests/rls-ddl.spec.ts b/src/tests/rls-ddl.spec.ts index 80e10cf6..5f092139 100644 --- a/src/tests/rls-ddl.spec.ts +++ b/src/tests/rls-ddl.spec.ts @@ -42,6 +42,8 @@ describe('RLS policy DDL & storage', () => { }); it('captures AS / FOR / TO / WITH CHECK', () => { + // the grantees must exist, as in postgres ("role ... does not exist") + none(`create role alice; create role bob`); none(`create policy p2 on docs as restrictive for insert to alice, bob with check (owner = current_user)`); const p = rls().policies.find((x: any) => x.name === 'p2'); expect(p.permissive).toBe(false); diff --git a/src/transaction.ts b/src/transaction.ts index 2683dd52..6a142902 100644 --- a/src/transaction.ts +++ b/src/transaction.ts @@ -17,8 +17,57 @@ export class Transaction implements _Transaction { return !!this.parent; } - private constructor(private parent: Transaction | null, private data: ImMap) { + aborted = false; + + /** puts the schema back as it was before this transaction's first DDL (see schema-snapshot.ts) */ + schemaRestore?: () => void; + + /** + * Called before a DDL statement runs in this transaction: this transaction and every enclosing + * one that has not seen DDL yet record the current schema, so whichever of them rolls back + * restores it. They all share one capture - the schema is the same for all of them right now. + */ + checkpointSchema(capture: () => () => void): void { + let restore: (() => void) | undefined; + const get = () => restore ??= capture(); + for (let x: Transaction | null = this; x && x.isChild; x = x.parent) { + if (!x.schemaRestore) { + x.schemaRestore = get(); + } + // savepoints declared since the last DDL capture the schema now, lazily: most + // savepoints see no DDL, and capturing at SAVEPOINT copied the whole schema each time + for (const [k, v] of x.savepointSchemas) { + if (!v) { + x.savepointSchemas.set(k, get()); + } + } + } + } + + /** the BEGIN block this transaction belongs to, if any */ + get explicitBlock(): Transaction | null { + for (let x: Transaction | null = this; x; x = x.parent) { + if (x.explicit) { + return x; + } + } + return null; + } + + get inExplicitBlock(): boolean { + return this.explicit || !!this.parent?.inExplicitBlock; + } + + /** + * When this transaction started - what now(), current_timestamp and current_date return for + * its whole lifetime in postgres. A transaction forked from root starts now; nested ones + * (BEGIN inside the implicit transaction of a call, savepoint-like children) inherit it. + */ + readonly startedAt: Date; + + private constructor(private parent: Transaction | null, private data: ImMap, private explicit = false) { this.origData = data; + this.startedAt = parent?.isChild ? parent.startedAt : new Date(); } @@ -26,8 +75,8 @@ export class Transaction implements _Transaction { return new Transaction(null, this.data); } - fork(): _Transaction { - return new Transaction(this, this.data); + fork(explicit = false): _Transaction { + return new Transaction(this, this.data, explicit); } commit(): _Transaction { @@ -35,7 +84,26 @@ export class Transaction implements _Transaction { return this; } if (this.parent.data !== this.origData) { - throw new NotSupported('Concurrent transactions'); + // the parent moved on while we were open - e.g. CREATE SCHEMA registers its catalogue + // tables straight into root. Rebase our changes onto it, key by key; only the same + // key changed on both sides is a real conflict. + let merged = this.parent.data; + for (const [k, v] of this.data) { + if (this.origData.get(k) === v) { + continue; + } + if (merged.get(k) !== this.origData.get(k)) { + throw new NotSupported('Concurrent transactions'); + } + merged = merged.set(k, v); + } + for (const k of this.origData.keys()) { + if (!this.data.has(k)) { + merged = merged.delete(k); + } + } + this.parent.data = merged; + return this.parent; } this.parent.data = this.data; return this.parent; @@ -49,20 +117,41 @@ export class Transaction implements _Transaction { } rollback() { + this.schemaRestore?.(); return this.parent ?? this; } + /** the outermost transaction of this call or block: discarding it undoes everything */ + discardAll(): void { + let outer: Transaction | null = null; + for (let x: Transaction | null = this; x && x.isChild; x = x.parent) { + if (x.schemaRestore) { + outer = x; + } + } + outer?.schemaRestore?.(); + } + savepoint(name: string): void { // re-declaring a name captures the current state under it (postgres hides the // older savepoint of the same name; we simply overwrite - close enough for v1) + this.savepoints.delete(name); // re-inserted last: it is now the newest savepoint this.savepoints.set(name, this.data); + // the schema is captured by the first DDL that follows (see checkpointSchema) + this.savepointSchemas.set(name, null); } + /** savepoint → schema restore, or null while no DDL has run since it was declared */ + private savepointSchemas = new Map void) | null>(); + rollbackTo(name: string): void { const saved = this.savepoints.get(name); if (saved === undefined) { throw new QueryError(`savepoint "${name}" does not exist`); } + this.savepointSchemas.get(name)?.(); + // the schema is back to the savepoint's: the next DDL captures it afresh + this.savepointSchemas.set(name, null); this.data = saved; // the savepoint survives (can be rolled back to again), but any savepoints // established after it are discarded @@ -83,11 +172,13 @@ export class Transaction implements _Transaction { reached = true; if (inclusive) { this.savepoints.delete(k); + this.savepointSchemas.delete(k); } continue; } if (reached) { this.savepoints.delete(k); + this.savepointSchemas.delete(k); } } } diff --git a/src/utils.ts b/src/utils.ts index dc563e92..f26b2cf3 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -82,6 +82,12 @@ export function watchUse(rootValue: T): { checked: T; check?: () => string | enumerable: true, }); } + // non-enumerable metadata (e.g. a numeric literal's text as written) rides along unwatched + for (const k of Object.getOwnPropertyNames(value)) { + if (!Object.prototype.propertyIsEnumerable.call(value, k)) { + Object.defineProperty(ret, k, { value: value[k], enumerable: false }); + } + } return ret; } @@ -380,6 +386,11 @@ export function executionCtx(): ExecCtx { } return curCtx[curCtx.length - 1]; } +/** now() in postgres: the current transaction's start time (wall clock outside of any execution) */ +export function transactionNow(): Date { + return curCtx.length ? new Date(curCtx[curCtx.length - 1].transaction.startedAt.getTime()) : new Date(); +} + export function hasExecutionCtx(): boolean { return curCtx.length > 0; } diff --git a/tools/corpus-diff/README.md b/tools/corpus-diff/README.md new file mode 100644 index 00000000..9cc5d86b --- /dev/null +++ b/tools/corpus-diff/README.md @@ -0,0 +1,54 @@ +# corpus-diff: pg-mem vs Postgres on real project migrations + +Runs real RapidNative project migrations through pg-mem and PGlite (real Postgres in WASM) side by +side and reports every place they disagree. PGlite is the oracle: any difference is a pg-mem gap. + +```bash +bun tools/corpus-diff/run.ts [outDir] [--only p07] +``` + +`//migrations/*.sql` plus an optional `seed.sql`, one directory per project. +Results land in `outDir/diffs.json` (default `/../corpus-diff-out`). + +**The corpus is user project code. Never commit it.** Pull it on demand into a scratch directory. + +## Getting a corpus + +From `rapidnative-website`, with the read-only BI credentials (see its `bi` skill). This picks a +seeded random sample of recent projects that have migrations: + +```sql +select setseed(0.20261006); +select p.id from projects p +where p.created_at >= now() - interval '30 days' + and exists (select 1 from files f where f.project_id = p.id and f.file_path like 'supabase/migrations/%.sql') +order by random() limit 50; +-- then per project: +select file_path, content from files where project_id = $1 + and (file_path like 'supabase/migrations/%.sql' or file_path = 'supabase/seed.sql') order by file_path; +``` + +Use the BI database connection, not the Supabase REST URL in the website's `.env`. That URL is a +different project and holds only a handful of rows. + +## What is compared + +Each project gets the same Supabase bootstrap the agent's validator installs (`bootstrap.ts`, copied +from rapidnative-website `pg-session.ts`), then, on both engines: + +| kind | what | +|---|---| +| `bootstrap`, `migration-*`, `seed-*` | each file applies, or fails, on both. `-rejected`: pg-mem refused SQL Postgres accepts (an app that won't boot). `-accepted`: pg-mem took SQL Postgres refuses (a validator waving through a broken migration) | +| `catalog` | the exact `information_schema` / `pg_*` reads `PgSession.buildTableInfo()` makes | +| `data` | rows after the seed (generated uuids and `now()`-relative timestamps normalised) | +| `rls-read`, `rls-write` | counts and affected rows as anon, the demo user and a stranger | +| `insert`, `reject` | a copy of a seeded row must insert; single-field corruptions (bad literal, NULL into NOT NULL, int overflow, dangling FK, duplicate key, CHECK violation) must fail like Postgres | +| `ddl` | policies with mistyped operands or missing columns, retyping/dropping columns other objects depend on, triggers on missing functions | +| `trigger` | `updated_at` triggers fire | +| `cascade` | deleting the demo user: what survives | +| `rollback` | state leaking past a ROLLBACK | + +Probes run in a transaction that is rolled back. pg-mem is rebuilt after DDL probes, because its +DDL is not transactional. + +To cluster `diffs.json` by root cause, group on `kind`, `probe`, and the normalised error text of each side. diff --git a/tools/corpus-diff/bootstrap.ts b/tools/corpus-diff/bootstrap.ts new file mode 100644 index 00000000..3b131fa1 --- /dev/null +++ b/tools/corpus-diff/bootstrap.ts @@ -0,0 +1,80 @@ +// The Supabase surface RapidNative's agent validator (rapidnative-website +// src/lib/coding-agent/db/pg-session.ts, AUTH_BOOTSTRAP) installs before applying a project's +// migrations. Copied verbatim so both engines are judged on exactly what production runs. +export const DEMO_USER_ID = '00000000-0000-0000-0000-000000000001'; +export const STRANGER_ID = '00000000-0000-0000-0000-0000000000ff'; + +export const AUTH_BOOTSTRAP = [ + `create schema if not exists auth`, + `create table if not exists auth.users ( + id uuid primary key default gen_random_uuid(), + aud text default 'authenticated', + role text default 'authenticated', + email text unique, + encrypted_password text, + email_confirmed_at timestamptz, + last_sign_in_at timestamptz, + raw_app_meta_data jsonb default '{}'::jsonb, + raw_user_meta_data jsonb default '{}'::jsonb, + is_super_admin boolean default false, + created_at timestamptz default now(), + updated_at timestamptz default now(), + phone text unique, + phone_confirmed_at timestamptz, + banned_until timestamptz, + deleted_at timestamptz, + is_anonymous boolean default false + )`, + `create function auth.uid() returns uuid as $$ + select coalesce(nullif(current_setting('request.jwt.claim.sub', true), ''), '${DEMO_USER_ID}')::uuid + $$ language sql stable`, + `create function auth.role() returns text as $$ + select coalesce(nullif(current_setting('request.jwt.claim.role', true), ''), 'authenticated')::text + $$ language sql stable`, + `create role anon nologin`, + `create role authenticated nologin`, + `grant usage on schema public to anon, authenticated`, + `alter default privileges in schema public grant select, insert, update, delete on tables to anon, authenticated`, + `create schema if not exists storage`, + `create table if not exists storage.buckets ( + id text primary key, + name text not null unique, + owner uuid, + owner_id text, + public boolean default false, + avif_autodetection boolean default false, + file_size_limit bigint, + allowed_mime_types text[], + created_at timestamptz default now(), + updated_at timestamptz default now() + )`, + `create table if not exists storage.objects ( + id uuid primary key default gen_random_uuid(), + bucket_id text references storage.buckets(id), + name text, + owner uuid, + owner_id text, + metadata jsonb, + path_tokens text[] generated always as (string_to_array(name, '/')) stored, + version text, + user_metadata jsonb, + created_at timestamptz default now(), + updated_at timestamptz default now(), + last_accessed_at timestamptz default now() + )`, + `alter table storage.objects enable row level security`, + `create function storage.foldername(name text) returns text[] as $$ + select (string_to_array(name, '/'))[1:array_length(string_to_array(name, '/'), 1) - 1] + $$ language sql immutable`, + `create function storage.filename(name text) returns text as $$ + select (string_to_array(name, '/'))[array_length(string_to_array(name, '/'), 1)] + $$ language sql immutable`, + `create function storage.extension(name text) returns text as $$ + select reverse(split_part(reverse((string_to_array(name, '/'))[array_length(string_to_array(name, '/'), 1)]), '.', 1)) + $$ language sql immutable`, + `grant usage on schema storage to anon, authenticated`, + `grant select, insert, update, delete on storage.buckets, storage.objects to anon, authenticated`, +]; + +/** PgSession.refresh() runs this after every schema change. */ +export const REFRESH_GRANT = `grant select, insert, update, delete on all tables in schema public to anon, authenticated`; diff --git a/tools/corpus-diff/engines.ts b/tools/corpus-diff/engines.ts new file mode 100644 index 00000000..dca9f825 --- /dev/null +++ b/tools/corpus-diff/engines.ts @@ -0,0 +1,68 @@ +// Two engines behind one async interface, so every step of the corpus diff runs the same SQL on +// both. PGlite is real Postgres compiled to WASM and is the source of truth; pg-mem is under test. +import { PGlite } from '@electric-sql/pglite'; +import { uuid_ossp } from '@electric-sql/pglite/contrib/uuid_ossp'; +import { pgcrypto } from '@electric-sql/pglite/contrib/pgcrypto'; +import { newDb } from '../../src'; + +export interface Outcome { + ok: boolean; + rows?: any[]; + rowCount?: number; + err?: string; +} + +export interface Engine { + kind: 'pglite' | 'pgmem'; + /** multi-statement, no results */ + exec(sql: string): Promise; + /** single statement with rows + affected count */ + query(sql: string): Promise<{ rows: any[]; rowCount: number }>; + close(): Promise; +} + +export async function createPglite(): Promise { + const db = new PGlite({ extensions: { uuid_ossp, pgcrypto } }); + await db.waitReady; + return { + kind: 'pglite', + exec: async (sql) => { + await db.exec(sql); + }, + query: async (sql) => { + const r = await db.query(sql); + return { rows: r.rows as any[], rowCount: r.affectedRows ?? r.rows.length }; + }, + close: () => db.close(), + }; +} + +export async function createPgmem(): Promise { + const db = newDb(); + return { + kind: 'pgmem', + exec: async (sql) => { + db.public.none(sql); + }, + query: async (sql) => { + const r = db.public.query(sql); + return { rows: r.rows ?? [], rowCount: r.rowCount ?? r.rows?.length ?? 0 }; + }, + close: async () => {}, + }; +} + +export const create = (kind: Engine['kind']) => (kind === 'pglite' ? createPglite() : createPgmem()); + +/** First line of an error, as PgSession records it. */ +export const errLine = (e: any) => String(e?.message ?? e).split('\n')[0]; + +export async function attempt(fn: () => Promise): Promise { + try { + const r = await fn(); + if (r && typeof r === 'object' && 'rows' in r) return { ok: true, rows: r.rows, rowCount: r.rowCount }; + return { ok: true }; + } catch (e) { + return { ok: false, err: errLine(e) }; + } +} diff --git a/tools/corpus-diff/run.ts b/tools/corpus-diff/run.ts new file mode 100644 index 00000000..0580550b --- /dev/null +++ b/tools/corpus-diff/run.ts @@ -0,0 +1,497 @@ +// Differential test of pg-mem against PGlite (real Postgres) on real project migrations. +// +// bun tools/corpus-diff/run.ts [outDir] [--only p07] +// +// //migrations/*.sql + optional seed.sql — one directory per project. Nothing +// from the corpus is committed: it is user project code, pulled on demand (see README.md). +// +// Every step runs on both engines and the outcomes are compared. PGlite is the oracle: a diff is a +// pg-mem gap, in either direction — rejecting SQL Postgres accepts (an app that will not boot), or +// accepting SQL Postgres rejects (a validator that waves through a migration that fails on publish). +import fs from 'node:fs'; +import path from 'node:path'; +import { AUTH_BOOTSTRAP, DEMO_USER_ID, REFRESH_GRANT, STRANGER_ID } from './bootstrap'; +import { attempt, create, type Engine, type Outcome } from './engines'; + +interface Project { + name: string; + migrations: { path: string; sql: string }[]; + seed?: string; + /** every literal in the project's SQL, used to tell seeded uuids from generated ones */ + text: string; +} + +interface Diff { + project: string; + kind: string; + table?: string; + probe: string; + sql?: string; + pglite: Outcome | unknown; + pgmem: Outcome | unknown; +} + +// ── loading ──────────────────────────────────────────────────────────── + +function loadProject(dir: string): Project { + const mdir = path.join(dir, 'migrations'); + const migrations = fs.existsSync(mdir) + ? fs.readdirSync(mdir).filter((f) => f.endsWith('.sql')).map((f) => ({ path: `supabase/migrations/${f}`, sql: fs.readFileSync(path.join(mdir, f), 'utf8') })) + : []; + const seedPath = path.join(dir, 'seed.sql'); + const seed = fs.existsSync(seedPath) ? fs.readFileSync(seedPath, 'utf8') : undefined; + return { name: path.basename(dir), migrations, seed, text: migrations.map((m) => m.sql).join('\n') + '\n' + (seed ?? '') }; +} + +// ── build: what PgSession.create() + db_seed do ─────────────────────────── + +interface Built { + eng: Engine; + /** wall clock when the seed ran: seed timestamps are now() ± an interval, compared relative to it */ + seedAt: number; + bootstrap: Outcome[]; + migrations: (Outcome & { path: string })[]; + seed?: Outcome; +} + +async function build(kind: Engine['kind'], p: Project): Promise { + const eng = await create(kind); + const bootstrap: Outcome[] = []; + for (const stmt of AUTH_BOOTSTRAP) bootstrap.push(await attempt(() => eng.exec(stmt))); + const migrations: Built['migrations'] = []; + for (const m of [...p.migrations].sort((a, b) => a.path.localeCompare(b.path))) { + migrations.push({ path: m.path, ...(await attempt(() => eng.exec(m.sql))) }); + } + await attempt(() => eng.exec(REFRESH_GRANT)); + const seedAt = Date.now(); + const seed = p.seed?.trim() ? await attempt(() => eng.exec(p.seed!)) : undefined; + return { eng, seedAt, bootstrap, migrations, seed }; +} + +// ── catalogue: exactly the reads PgSession.buildTableInfo() makes ───────── + +const catalogQueries = (t: string) => ({ + columns: `select column_name, data_type, is_nullable, column_default from information_schema.columns + where table_schema = 'public' and table_name = '${t}'`, + rls: `select rowsecurity from pg_tables where schemaname = 'public' and tablename = '${t}'`, + policies: `select * from pg_policies where tablename = '${t}'`, + indexes: `select indexname, indexdef from pg_indexes where schemaname = 'public' and tablename = '${t}'`, + foreignKeys: `select tc.constraint_name, kcu.column_name, rc.delete_rule + from information_schema.table_constraints tc + join information_schema.key_column_usage kcu + on kcu.constraint_name = tc.constraint_name + left join information_schema.referential_constraints rc + on rc.constraint_name = tc.constraint_name + where tc.constraint_type = 'FOREIGN KEY' and tc.table_name = '${t}'`, + triggers: `select trigger_name, action_timing, event_manipulation + from information_schema.triggers where event_object_table = '${t}'`, + primaryKey: `select kcu.column_name + from information_schema.table_constraints tc + join information_schema.key_column_usage kcu + on kcu.constraint_name = tc.constraint_name + where tc.constraint_type = 'PRIMARY KEY' and tc.table_name = '${t}'`, +}); + +/** Normalise a column default so cosmetic deparse differences ('x'::text vs 'x') don't count. */ +/** drop one paren pair only when it wraps the whole expression ("(a)+(b)" is left alone) */ +const unwrap = (x: string) => { + if (!x.startsWith('(') || !x.endsWith(')')) return x; + let depth = 0; + for (let i = 0; i < x.length; i++) { + if (x[i] === '(') depth++; + else if (x[i] === ')' && --depth === 0 && i < x.length - 1) return x; + } + return x.slice(1, -1); +}; +const normDefault = (d: any) => + d == null ? null : unwrap(String(d).toLowerCase().replace(/::[a-z_ ]+(\[\])?/g, '').replace(/\s+/g, '').replace(/"/g, '')); + +const sortBy = (xs: T[], key: (x: T) => string) => [...xs].sort((a, b) => (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0)); + +/** Each catalogue facet reduced to the fields PgSession actually reads. */ +const shapeCatalog: Record any> = { + columns: (rows) => sortBy(rows.map((r) => ({ name: r.column_name, type: r.data_type, nullable: String(r.is_nullable).toUpperCase() === 'YES', default: normDefault(r.column_default) })), (c) => c.name), + rls: (rows) => !!rows[0]?.rowsecurity, + policies: (rows) => sortBy(rows.map((r) => ({ name: r.policyname, cmd: r.cmd ?? 'ALL', using: r.qual != null, check: r.with_check != null })), (p) => p.name), + indexes: (rows) => rows.map((r) => r.indexname).sort(), + foreignKeys: (rows) => sortBy(rows.map((r) => ({ column: r.column_name, onDelete: r.delete_rule ?? 'NO ACTION' })), (f) => f.column + f.onDelete), + triggers: (rows) => [...new Set(rows.map((r) => `${r.trigger_name}:${r.action_timing}:${r.event_manipulation}`))].sort(), + primaryKey: (rows) => rows.map((r) => String(r.column_name)).sort(), +}; + +// ── value normalisation for row comparison ────────────────────────────── + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function normValue(v: any, p: Project, startedAt: number): any { + if (v == null) return null; + if (v instanceof Date) { + // timestamps made from now() during the run differ between engines by the ms the runs are + // apart: compare them as an offset from that engine's own seed time, to the nearest 10s + const off = v.getTime() - startedAt; + return Math.abs(off) < 400 * 86400_000 ? `= 0 ? '+' : ''}${Math.round(off / 10_000) * 10}s>` : v.toISOString(); + } + if (typeof v === 'bigint') return Number(v); + if (typeof v === 'number') return Number.isInteger(v) ? v : Math.round(v * 1e6) / 1e6; + if (typeof v === 'string') { + if (UUID.test(v)) return p.text.toLowerCase().includes(v.toLowerCase()) ? v.toLowerCase() : ''; + if (/^-?\d+(\.\d+)?$/.test(v)) return normValue(Number(v), p, startedAt); // numeric comes back as a string from PGlite + if (/^\d{4}-\d\d-\d\d[ T]\d\d:\d\d/.test(v)) { + const t = Date.parse(v.replace(' ', 'T')); + if (!Number.isNaN(t)) return normValue(new Date(t), p, startedAt); + } + return v; + } + if (Array.isArray(v)) return v.map((x) => normValue(x, p, startedAt)); + if (typeof v === 'object') return Object.fromEntries(Object.keys(v).sort().map((k) => [k, normValue(v[k], p, startedAt)])); + return v; +} + +const normRows = (rows: any[] | undefined, p: Project, t0: number) => + (rows ?? []).map((r) => JSON.stringify(normValue(r, p, t0))).sort(); + +const same = (a: any, b: any) => JSON.stringify(a) === JSON.stringify(b); + +// ── SQL literal rendering (from PGlite's rows, so both engines get identical text) ── + +const q = (s: string) => `'${s.replace(/'/g, "''")}'`; +const ident = (s: string) => `"${s.replace(/"/g, '""')}"`; + +function literal(v: any, type: string): string { + if (v == null) return 'null'; + if (v instanceof Date) return q(v.toISOString()); + if (typeof v === 'boolean') return v ? 'true' : 'false'; + if (typeof v === 'number' || typeof v === 'bigint') return String(v); + if (type === 'ARRAY' && Array.isArray(v)) return `${q('{' + v.map((x) => (x == null ? 'NULL' : '"' + String(x).replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"')).join(',') + '}')}`; + if (type === 'json' || type === 'jsonb') return q(JSON.stringify(v)); + if (typeof v === 'object') return q(JSON.stringify(v)); + return q(String(v)); +} + +/** A fresh value for a unique/PK column, so a template insert does not collide. */ +function perturb(v: any, type: string): string { + if (type === 'uuid') return q(crypto.randomUUID()); + if (/int|numeric|real|double/.test(type)) return String(Math.floor(Number(v ?? 0)) + 1_000_000 + Math.floor(Math.random() * 1000)); + if (type === 'text' || type.startsWith('character')) return q(`${v ?? 'x'}-probe${Math.floor(Math.random() * 1e6)}`); + return literal(v, type); +} + +// ── probes ─────────────────────────────────────────────────────────── + +interface Probe { + kind: string; + table: string; + probe: string; + /** statements; the last one's outcome is compared */ + sql: string[]; + role?: { as: 'anon' | 'authenticated'; uid: string }; + ddl?: boolean; + /** compare returned rows, not just ok/rowCount */ + rows?: boolean; +} + +const asRole = (r: NonNullable) => [ + `select set_config('request.jwt.claim.sub', '${r.uid}', false)`, + `select set_config('request.jwt.claim.role', '${r.as}', false)`, + `set role ${r.as}`, +]; + +const DEMO = { as: 'authenticated' as const, uid: DEMO_USER_ID }; +const STRANGER = { as: 'authenticated' as const, uid: STRANGER_ID }; +const ANON = { as: 'anon' as const, uid: '' }; + +interface TableMeta { + name: string; + cols: { name: string; type: string; nullable: boolean; hasDefault: boolean; generated: boolean }[]; + pk: string[]; + unique: string[][]; + fks: { cols: string[]; refTable: string }[]; + checks: { cols: string[]; def: string }[]; + policyCols: string[]; + referencedBy: number; + sample?: Record; +} + +/** Everything probe generation needs, read from the oracle. */ +async function tableMeta(eng: Engine, t: string): Promise { + const cols = (await eng.query(`select column_name, data_type, is_nullable, column_default, is_generated, is_identity + from information_schema.columns where table_schema='public' and table_name='${t}' order by ordinal_position`)).rows.map((r) => ({ + name: r.column_name, type: r.data_type, nullable: r.is_nullable === 'YES', hasDefault: r.column_default != null || r.is_identity === 'YES', generated: r.is_generated === 'ALWAYS', + })); + const cons = (await eng.query(`select c.contype, pg_get_constraintdef(c.oid) def, c.confrelid::regclass::text reftable, + array(select a.attname::text from unnest(c.conkey) k join pg_attribute a on a.attrelid=c.conrelid and a.attnum=k) cols + from pg_constraint c where c.conrelid = 'public.${ident(t)}'::regclass`)).rows; + const policies = (await eng.query(`select qual, with_check from pg_policies where schemaname='public' and tablename='${t}'`)).rows; + const policyText = policies.map((r) => `${r.qual ?? ''} ${r.with_check ?? ''}`).join(' '); + const referencedBy = Number((await eng.query(`select count(*)::int n from pg_constraint where contype='f' and confrelid = 'public.${ident(t)}'::regclass and conrelid <> confrelid`)).rows[0].n); + const sample = (await eng.query(`select * from public.${ident(t)} limit 1`)).rows[0]; + return { + name: t, + cols, + pk: cons.find((c) => c.contype === 'p')?.cols ?? [], + unique: cons.filter((c) => c.contype === 'u').map((c) => c.cols), + fks: cons.filter((c) => c.contype === 'f').map((c) => ({ cols: c.cols, refTable: c.reftable })), + checks: cons.filter((c) => c.contype === 'c').map((c) => ({ cols: c.cols, def: c.def })), + policyCols: cols.map((c) => c.name).filter((n) => new RegExp(`\\b${n}\\b`).test(policyText)), + referencedBy, + sample, + }; +} + +function templateInsert(m: TableMeta, override: Record = {}): string | null { + if (!m.sample) return null; + const keyCols = new Set([...m.pk, ...m.unique.flat()]); + const cols = m.cols.filter((c) => !c.generated); + const vals = cols.map((c) => { + if (c.name in override) return override[c.name]; + if (keyCols.has(c.name)) return perturb(m.sample![c.name], c.type); + return literal(m.sample![c.name], c.type); + }); + return `insert into public.${ident(m.name)} (${cols.map((c) => ident(c.name)).join(', ')}) values (${vals.join(', ')})`; +} + +/** + * A key value both engines share: one written in the project's SQL. A generated one + * (gen_random_uuid(), serial) differs between the engines, so probing for it on pg-mem with + * PGlite's value would compare two different rows. + */ +const sharedKey = (v: any, text: string) => v != null && (typeof v === 'number' ? text.includes(String(v)) : text.toLowerCase().includes(String(v).toLowerCase())); + +function probesFor(m: TableMeta, all: TableMeta[], text: string): Probe[] { + const t = ident(m.name); + const out: Probe[] = []; + const P = (p: Omit) => out.push({ table: m.name, ...p }); + + // RLS reads + for (const [label, role] of [['anon', ANON], ['demo', DEMO], ['stranger', STRANGER]] as const) { + P({ kind: 'rls-read', probe: `count as ${label}`, sql: [`select count(*)::int as n from public.${t}`], role, rows: true }); + } + const anyCol = m.cols.find((c) => !c.generated && !m.pk.includes(c.name))?.name ?? m.cols[0]?.name; + if (anyCol) { + for (const [label, role] of [['demo', DEMO], ['stranger', STRANGER], ['anon', ANON]] as const) { + P({ kind: 'rls-write', probe: `update as ${label}`, sql: [`update public.${t} set ${ident(anyCol)} = ${ident(anyCol)}`], role }); + P({ kind: 'rls-write', probe: `delete as ${label}`, sql: [`delete from public.${t}`], role }); + } + } + // the same through an index lookup (WHERE = ): index paths must not skip RLS + if (m.sample && m.pk.length === 1 && sharedKey(m.sample[m.pk[0]], text)) { + const pkCol = m.cols.find(c => c.name === m.pk[0])!; + const where = `where ${ident(pkCol.name)} = ${literal(m.sample[pkCol.name], pkCol.type)}`; + for (const [label, role] of [['anon', ANON], ['demo', DEMO], ['stranger', STRANGER]] as const) { + P({ kind: 'rls-read', probe: `pk lookup as ${label}`, sql: [`select count(*)::int as n from public.${t} ${where}`], role, rows: true }); + if (anyCol) { + P({ kind: 'rls-write', probe: `update by pk as ${label}`, sql: [`update public.${t} set ${ident(anyCol)} = ${ident(anyCol)} ${where}`], role }); + P({ kind: 'rls-write', probe: `delete by pk as ${label}`, sql: [`delete from public.${t} ${where}`], role }); + } + } + } + // updated_at trigger: inside one transaction now() is constant, so a row the trigger touched has updated_at = now() + if (m.sample && m.cols.some((c) => c.name === 'updated_at') && anyCol && anyCol !== 'updated_at') { + P({ kind: 'trigger', probe: 'updated_at fires on update', sql: [`update public.${t} set ${ident(anyCol)} = ${ident(anyCol)} returning (updated_at = now()) as fired`], rows: true }); + } + + // template insert (positive) and single-field corruptions (negative), superuser and per role + const tmpl = templateInsert(m); + if (tmpl) { + P({ kind: 'insert', probe: 'template row', sql: [tmpl] }); + P({ kind: 'rls-write', probe: 'template insert as demo', sql: [tmpl], role: DEMO }); + P({ kind: 'rls-write', probe: 'template insert as stranger', sql: [tmpl], role: STRANGER }); + P({ kind: 'rls-write', probe: 'template insert as anon', sql: [tmpl], role: ANON }); + for (const c of m.cols.filter((c) => !c.generated)) { + const bad: [string, string][] = []; + if (!c.nullable) bad.push(['null into not-null', 'null']); + if (/^(integer|bigint|smallint|numeric|real|double precision)$/.test(c.type)) bad.push(['non-numeric literal', q('abc')]); + if (c.type === 'uuid') bad.push(['malformed uuid', q('not-a-uuid')]); + if (c.type === 'boolean') bad.push(['malformed boolean', q('maybe')]); + if (/^(timestamp|date|time)/.test(c.type)) bad.push(['malformed timestamp', q('not-a-date')]); + if (c.type === 'jsonb' || c.type === 'json') bad.push(['malformed json', q('{bad')]); + if (c.type === 'integer') bad.push(['int4 overflow', '3000000000']); + if (c.type === 'smallint') bad.push(['int2 overflow', '70000']); + if (/^character varying/.test(c.type)) bad.push(['varchar too long (if bounded)', q('x'.repeat(10_001))]); + if (m.fks.some((f) => f.cols.includes(c.name))) bad.push(['dangling foreign key', c.type === 'uuid' ? q(crypto.randomUUID()) : q('no-such-parent')]); + if (m.sample && [m.pk, ...m.unique].some((k) => k.length === 1 && k[0] === c.name) && sharedKey(m.sample[c.name], text)) bad.push(['duplicate key', literal(m.sample[c.name], c.type)]); + for (const ch of m.checks.filter((ch) => ch.cols.includes(c.name))) { + if (c.type === 'text' || c.type.startsWith('character')) bad.push([`check violation ${ch.def.slice(0, 60)}`, q('__bogus__')]); + else if (/int|numeric|real|double/.test(c.type)) bad.push([`check violation ${ch.def.slice(0, 60)}`, '-999999']); + } + for (const [what, val] of bad) { + const sql = templateInsert(m, { [c.name]: val }); + if (sql) P({ kind: 'reject', probe: `${c.name}: ${what}`, sql: [sql] }); + } + } + } + + // DDL the validator must judge the way Postgres does + for (const c of m.cols) { + if (c.type === 'text' || c.type === 'uuid') { + P({ kind: 'ddl', probe: `policy auth.uid() = ${c.type} column (${c.name})`, sql: [`create policy zz_probe on public.${t} for select using (auth.uid() = ${ident(c.name)})`], ddl: true }); + } + } + P({ kind: 'ddl', probe: 'policy on missing column', sql: [`create policy zz_probe on public.${t} for select using (zz_no_such_col = 1)`], ddl: true }); + for (const c of m.policyCols) { + P({ kind: 'ddl', probe: `alter type of policy-referenced column ${c}`, sql: [`alter table public.${t} alter column ${ident(c)} type text`], ddl: true }); + } + P({ kind: 'ddl', probe: 'index on missing column', sql: [`create index zz_probe_idx on public.${t} (zz_no_such_col)`], ddl: true }); + if (m.cols[0]) P({ kind: 'ddl', probe: 'add existing column', sql: [`alter table public.${t} add column ${ident(m.cols[0].name)} text`], ddl: true }); + P({ kind: 'ddl', probe: 'create existing table', sql: [`create table public.${t} (id int)`], ddl: true }); + P({ kind: 'ddl', probe: 'create table if not exists (existing)', sql: [`create table if not exists public.${t} (id int)`], ddl: true }); + if (m.referencedBy > 0) P({ kind: 'ddl', probe: 'drop table with dependents', sql: [`drop table public.${t}`], ddl: true }); + if (m.pk.length === 1 && m.referencedBy > 0) P({ kind: 'ddl', probe: 'drop referenced pk column', sql: [`alter table public.${t} drop column ${ident(m.pk[0])}`], ddl: true }); + P({ kind: 'ddl', probe: 'drop policy if exists (missing)', sql: [`drop policy if exists zz_nope on public.${t}`], ddl: true }); + P({ kind: 'ddl', probe: 'drop policy (missing)', sql: [`drop policy zz_nope on public.${t}`], ddl: true }); + P({ kind: 'ddl', probe: 'trigger with missing function', sql: [`create trigger zz_trg before update on public.${t} for each row execute function public.zz_no_such_fn()`], ddl: true }); + void all; + return out; +} + +/** Cross-table probes. */ +function globalProbes(metas: TableMeta[]): Probe[] { + const out: Probe[] = []; + if (!metas.length) return out; + const counts = `select ${metas.map((m) => `(select count(*)::int from public.${ident(m.name)}) as ${ident(m.name)}`).join(', ')}`; + out.push({ kind: 'cascade', table: '*', probe: 'delete demo user, surviving rows', sql: [`delete from auth.users where id = '${DEMO_USER_ID}'`, counts], rows: true }); + out.push({ kind: 'reject', table: 'auth.users', probe: 'pg uuid = text comparison', sql: [`select 1 where auth.uid() = 'abc'::text`] }); + return out; +} + +// ── running a probe on one engine ───────────────────────────────────────── + +async function runProbe(eng: Engine, p: Probe): Promise { + await attempt(() => eng.exec('begin')); + try { + if (p.role) for (const s of asRole(p.role)) await eng.exec(s); + let last: Outcome = { ok: true }; + for (const s of p.sql) { + last = await attempt(() => eng.query(s)); + if (!last.ok) break; + } + return last; + } finally { + await attempt(() => eng.exec('reset role')); + await attempt(() => eng.exec('rollback')); + } +} + +/** Row counts of every user table + auth.users — detects a rollback that did not roll back. */ +async function fingerprint(eng: Engine, tables: string[]): Promise { + const parts: string[] = []; + for (const t of ['auth.users', ...tables.map((t) => `public.${ident(t)}`)]) { + const o = await attempt(() => eng.query(`select count(*)::int as n from ${t}`)); + parts.push(`${t}=${o.ok ? o.rows![0].n : 'ERR'}`); + } + const zz = await attempt(() => eng.query(`select count(*)::int as n from pg_policies where policyname = 'zz_probe'`)); + parts.push(`zz=${zz.ok ? zz.rows![0].n : '?'}`); + return parts.join(','); +} + +// ── per project ─────────────────────────────────────────────────────── + +async function diffProject(p: Project): Promise<{ diffs: Diff[]; stats: Record }> { + const t0 = Date.now(); + const diffs: Diff[] = []; + const stats: Record = {}; + const bump = (k: string) => (stats[k] = (stats[k] ?? 0) + 1); + const D = (d: Omit) => diffs.push({ project: p.name, ...d }); + + const ref = await build('pglite', p); + let mem = await build('pgmem', p); + + ref.bootstrap.forEach((o, i) => { + if (o.ok !== mem.bootstrap[i].ok) D({ kind: 'bootstrap', probe: AUTH_BOOTSTRAP[i].replace(/\s+/g, ' ').slice(0, 80), pglite: o, pgmem: mem.bootstrap[i] }); + }); + ref.migrations.forEach((o, i) => { + bump('migrations'); + const m = mem.migrations[i]; + if (o.ok !== m.ok) D({ kind: o.ok ? 'migration-rejected' : 'migration-accepted', probe: o.path, pglite: o, pgmem: m }); + }); + if (ref.seed) { + bump('seeds'); + if (ref.seed.ok !== mem.seed!.ok) D({ kind: ref.seed.ok ? 'seed-rejected' : 'seed-accepted', probe: 'seed.sql', pglite: ref.seed, pgmem: mem.seed }); + } + + // catalogue + const tables = (await ref.eng.query(`select tablename from pg_tables where schemaname = 'public' order by 1`)).rows.map((r) => r.tablename as string); + const memTablesO = await attempt(() => mem.eng.query(`select tablename from pg_tables where schemaname = 'public' order by 1`)); + const memTables = memTablesO.ok ? memTablesO.rows!.map((r) => r.tablename as string).sort() : []; + if (!same(tables, memTables)) D({ kind: 'catalog', table: '*', probe: 'pg_tables', pglite: tables, pgmem: memTablesO.ok ? memTables : memTablesO }); + + for (const t of tables) { + for (const [facet, sql] of Object.entries(catalogQueries(t))) { + bump('catalog'); + const a = await attempt(() => ref.eng.query(sql)); + const b = await attempt(() => mem.eng.query(sql)); + if (!b.ok) { D({ kind: 'catalog', table: t, probe: `${facet} (query failed)`, sql, pglite: '(ok)', pgmem: b }); continue; } + const sa = shapeCatalog[facet](a.rows!); + const sb = shapeCatalog[facet](b.rows!); + if (!same(sa, sb)) D({ kind: 'catalog', table: t, probe: facet, pglite: sa, pgmem: sb }); + } + // data after seed + bump('data'); + const a = await attempt(() => ref.eng.query(`select * from public.${ident(t)}`)); + const b = await attempt(() => mem.eng.query(`select * from public.${ident(t)}`)); + const ra = normRows(a.rows, p, ref.seedAt), rb = normRows(b.rows, p, mem.seedAt); + if (a.ok !== b.ok || !same(ra, rb)) D({ kind: 'data', table: t, probe: 'rows after seed', pglite: a.ok ? ra.slice(0, 3) : a, pgmem: b.ok ? rb.slice(0, 3) : b }); + } + + // probes (generated from the oracle's catalogue) + const metas: TableMeta[] = []; + for (const t of tables) metas.push(await tableMeta(ref.eng, t)); + const probes = [...metas.flatMap((m) => probesFor(m, metas, p.text)), ...globalProbes(metas)]; + const memFp0 = await fingerprint(mem.eng, memTables); + for (const pr of probes) { + bump(`probe:${pr.kind}`); + const a = await runProbe(ref.eng, pr); + const b = await runProbe(mem.eng, pr); + let differ = a.ok !== b.ok; + if (!differ && a.ok && !pr.ddl) differ = (a.rowCount ?? 0) !== (b.rowCount ?? 0) && /^(update|delete|insert)/i.test(pr.sql.at(-1)!); + if (!differ && a.ok && pr.rows) differ = !same(normRows(a.rows, p, t0), normRows(b.rows, p, t0)); + if (differ) { + D({ kind: pr.kind, table: pr.table, probe: pr.probe, sql: pr.sql.join(';\n'), pglite: { ...a, rows: a.rows?.slice(0, 3) }, pgmem: { ...b, rows: b.rows?.slice(0, 3) } }); + } + // pg-mem cannot roll back DDL; rebuild instead of letting one probe poison the next + const leaked = pr.ddl ? true : (await fingerprint(mem.eng, memTables)) !== memFp0; + if (leaked) { + if (!pr.ddl) D({ kind: 'rollback', table: pr.table, probe: `rollback after: ${pr.probe}`, pglite: '(restored)', pgmem: '(state leaked past rollback)' }); + await mem.eng.close(); + mem = await build('pgmem', p); + } + } + await ref.eng.close(); + await mem.eng.close(); + return { diffs, stats }; +} + +// ── main ────────────────────────────────────────────────────────────── + +const args = process.argv.slice(2); +const corpus = args[0]; +const outDir = args[1] && !args[1].startsWith('--') ? args[1] : path.join(corpus, '..', 'corpus-diff-out'); +const only = args.includes('--only') ? args[args.indexOf('--only') + 1] : null; +if (!corpus) { + console.error('usage: bun tools/corpus-diff/run.ts [outDir] [--only ]'); + process.exit(2); +} +fs.mkdirSync(outDir, { recursive: true }); +const projects = fs.readdirSync(corpus).filter((d) => fs.statSync(path.join(corpus, d)).isDirectory()).filter((d) => !only || d === only).sort(); +const all: Diff[] = []; +const totals: Record = {}; +const perProject: { project: string; diffs: number; ms: number; error?: string }[] = []; +for (const name of projects) { + const t = Date.now(); + try { + const { diffs, stats } = await diffProject(loadProject(path.join(corpus, name))); + all.push(...diffs); + for (const [k, v] of Object.entries(stats)) totals[k] = (totals[k] ?? 0) + v; + perProject.push({ project: name, diffs: diffs.length, ms: Date.now() - t }); + console.log(`${name}: ${diffs.length} diffs (${Date.now() - t}ms)`); + } catch (e: any) { + perProject.push({ project: name, diffs: -1, ms: Date.now() - t, error: String(e?.stack ?? e) }); + console.log(`${name}: HARNESS ERROR ${String(e?.message ?? e).split('\n')[0]}`); + } +} +fs.writeFileSync(path.join(outDir, 'diffs.json'), JSON.stringify({ totals, perProject, diffs: all }, null, 1)); +const byKind: Record = {}; +for (const d of all) byKind[d.kind] = (byKind[d.kind] ?? 0) + 1; +console.log('\ntotals', totals); +console.log('diffs by kind', byKind); +console.log(`projects with ≥1 diff: ${perProject.filter((p) => p.diffs !== 0).length}/${perProject.length}`);