diff --git a/packages/sync-rules/src/compiler/compiler.ts b/packages/sync-rules/src/compiler/compiler.ts index 50dadcf51..1bcf2228a 100644 --- a/packages/sync-rules/src/compiler/compiler.ts +++ b/packages/sync-rules/src/compiler/compiler.ts @@ -6,6 +6,7 @@ import { StreamOptions, SyncPlan } from '../sync_plan/plan.js'; import { CompilerModelToSyncPlan } from './ir_to_sync_plan.js'; import { QuerierGraphBuilder } from './querier_graph.js'; import { StreamQueryParser } from './parser.js'; +import { NodeLocations } from './expression.js'; /** * State for compiling sync streams. @@ -39,6 +40,7 @@ export class SyncStreamsCompiler { const parser = new StreamQueryParser({ compiler: this, originalText: sql, + locations: new NodeLocations(), errors }); const query = parser.parse(stmt); diff --git a/packages/sync-rules/src/compiler/expression.ts b/packages/sync-rules/src/compiler/expression.ts index 48f40a626..83950cda3 100644 --- a/packages/sync-rules/src/compiler/expression.ts +++ b/packages/sync-rules/src/compiler/expression.ts @@ -1,8 +1,12 @@ -import { Expr } from 'pgsql-ast-parser'; +import { Expr, NodeLocation, PGNode } from 'pgsql-ast-parser'; import { SourceResultSet } from './table.js'; import { EqualsIgnoringResultSet, equalsIgnoringResultSetList } from './compatibility.js'; import { StableHasher } from './equality.js'; import { ConnectionParameterSource } from '../sync_plan/plan.js'; +import { ExternalData, SqlExpression } from '../sync_plan/expression.js'; +import { ExpressionToSqlite } from '../sync_plan/expression_to_sql.js'; +import { RecursiveExpressionVisitor } from '../sync_plan/expression_visitor.js'; +import { getLocation } from '../errors.js'; /** * An analyzed SQL expression tracking dependencies on non-static data (i.e. rows or connection sources). @@ -16,23 +20,46 @@ import { ConnectionParameterSource } from '../sync_plan/plan.js'; * clauses) and to evaluate expressions at runtime (by preparing them as a statement and binding external values). */ export class SyncExpression implements EqualsIgnoringResultSet { + #sql?: string; + #instantiation?: readonly ExpressionInput[]; + + /** + * The original expression, where references to row or connection parameters have been replaced with SQL variables + * that are tracked through {@link instantiation}. + * + * This is only used to compute hash codes and to check instances for equality. {@link node} is the canonical + * representation of this expression. + */ + get sql(): string { + return (this.#sql ??= ExpressionToSqlite.toSqlite(this.node)); + } + + /** + * The values to instantiate parameters in {@link sqlExpression} with to retain original semantics of the + * expression. + */ + get instantiation(): readonly ExpressionInput[] { + if (this.#instantiation != null) { + return this.#instantiation; + } + + const instantiation: ExpressionInput[] = []; + FindExternalData.instance.visit(this.node, instantiation); + return (this.#instantiation = instantiation); + } + + get location(): NodeLocation { + return this.locations.locationFor(this.node); + } + constructor( - /** - * The original expression, where references to row or connection parameters have been replaced with SQL variables - * that are tracked through {@link instantiation}. - */ - readonly sql: string, /** * The AST node backing {@link sql}. * * We use this to be able to compose expressions, e.g. to possibly merge them. */ - readonly node: Expr, - /** - * The values to instantiate parameters in {@link sqlExpression} with to retain original semantics of the - * expression. - */ - readonly instantiation: ExpressionInputWithSpan[] + readonly node: SqlExpression, + readonly locations: NodeLocations ) {} equalsAssumingSameResultSet(other: EqualsIgnoringResultSet): boolean { @@ -47,32 +74,22 @@ export class SyncExpression implements EqualsIgnoringResultSet { hasher.addString(this.sql); equalsIgnoringResultSetList.hash(hasher, this.instantiation); } - - *instantiationValues() { - for (const instantiation of this.instantiation) { - yield instantiation.value; - } - } } -export type ExpressionInput = ColumnInRow | ConnectionParameter; - -export class ExpressionInputWithSpan implements EqualsIgnoringResultSet { - constructor( - readonly value: ExpressionInput, - readonly startOffset: number, - readonly length: number - ) {} - - equalsAssumingSameResultSet(other: EqualsIgnoringResultSet): boolean { - return other instanceof ExpressionInputWithSpan && other.value.equalsAssumingSameResultSet(this.value); +class FindExternalData extends RecursiveExpressionVisitor { + defaultExpression(expr: SqlExpression, arg: ExpressionInput[]): void { + this.visitChildren(expr, arg); } - assumingSameResultSetEqualityHashCode(hasher: StableHasher): void { - return this.value.assumingSameResultSetEqualityHashCode(hasher); + visitExternalData(expr: ExternalData, arg: ExpressionInput[]): void { + arg.push(expr.source); } + + static readonly instance: FindExternalData = new FindExternalData(); } +export type ExpressionInput = ColumnInRow | ConnectionParameter; + export class ColumnInRow implements EqualsIgnoringResultSet { constructor( readonly syntacticOrigin: Expr, @@ -103,3 +120,23 @@ export class ConnectionParameter implements EqualsIgnoringResultSet { hasher.addString(this.source); } } + +/** + * Tracks the original source location for translated {@link SqlExpression} nodes. + * + * We want to serialize translated expressions for sync plan, so embedding source offsets in them expands the size of + * sync plans and is tedious. We only need access to node locations while compiling sync streams, which we store in this + * in-memory map. + */ +export class NodeLocations { + readonly sourceForNode = new Map, PGNode | NodeLocation>(); + + locationFor(source: SqlExpression): NodeLocation { + const location = getLocation(this.sourceForNode.get(source)); + if (location == null) { + throw new Error('Missing location'); + } + + return location; + } +} diff --git a/packages/sync-rules/src/compiler/filter.ts b/packages/sync-rules/src/compiler/filter.ts index 092d52eed..70a86a6b7 100644 --- a/packages/sync-rules/src/compiler/filter.ts +++ b/packages/sync-rules/src/compiler/filter.ts @@ -45,7 +45,7 @@ export class SingleDependencyExpression implements EqualsIgnoringResultSet { constructor(expression: SyncExpression | SingleDependencyExpression) { if (expression instanceof SyncExpression) { - const checked = SingleDependencyExpression.extractSingleDependency(expression.instantiationValues()); + const checked = SingleDependencyExpression.extractSingleDependency(expression.instantiation); if (checked == null) { throw new InvalidExpressionError('Expression with multiple dependencies passed to SingleDependencyExpression'); } @@ -133,7 +133,7 @@ export class EqualsClause { ) {} get location(): NodeLocation | undefined { - return expandNodeLocations([this.left.expression.node, this.right.expression.node]); + return expandNodeLocations([this.left.expression.location, this.right.expression.location]); } } diff --git a/packages/sync-rules/src/compiler/filter_simplifier.ts b/packages/sync-rules/src/compiler/filter_simplifier.ts index 22fbb3818..5af8b5312 100644 --- a/packages/sync-rules/src/compiler/filter_simplifier.ts +++ b/packages/sync-rules/src/compiler/filter_simplifier.ts @@ -1,9 +1,8 @@ -import { assignChanged, astMapper, BinaryOperator, Expr } from 'pgsql-ast-parser'; import { And, BaseTerm, EqualsClause, isBaseTerm, Or, SingleDependencyExpression } from './filter.js'; -import { PostgresToSqlite } from './sqlite.js'; -import { ExpressionInput, SyncExpression } from './expression.js'; -import { expandNodeLocations } from '../errors.js'; +import { SyncExpression } from './expression.js'; import { SourceResultSet } from './table.js'; +import { BinaryOperator } from '../sync_plan/expression.js'; +import { expandNodeLocations } from '../errors.js'; export class FilterConditionSimplifier { constructor(private readonly originalText: string) {} @@ -21,7 +20,7 @@ export class FilterConditionSimplifier { } } - baseTerms = this.mergeByCommonDependencies('OR', baseTerms); + baseTerms = this.mergeByCommonDependencies('or', baseTerms); for (const term of baseTerms) { andTerms.push({ terms: [term] }); } @@ -30,7 +29,7 @@ export class FilterConditionSimplifier { } private simplifyAnd(and: And): And | BaseTerm { - const merged = this.mergeByCommonDependencies('AND', and.terms); + const merged = this.mergeByCommonDependencies('and', and.terms); if (merged.length == 1) { return merged[0]; @@ -92,8 +91,8 @@ export class FilterConditionSimplifier { // must be a row condition since it can't be represented as parameters that could be instantiated. if ( SingleDependencyExpression.extractSingleDependency([ - ...base.left.expression.instantiationValues(), - ...base.right.expression.instantiationValues() + ...base.left.expression.instantiation, + ...base.right.expression.instantiation ]) ) { return this.composeExpressions('=', base.left, base.right); @@ -109,52 +108,26 @@ export class FilterConditionSimplifier { * For instance, `composeExpressions('AND', a, b, c)` returns `a AND b AND c` as a single expression. All expressions * must have compatible dependencies. */ - private composeExpressions(op: BinaryOperator, ...terms: SingleDependencyExpression[]): SingleDependencyExpression { + private composeExpressions( + operator: BinaryOperator, + ...terms: SingleDependencyExpression[] + ): SingleDependencyExpression { if (terms.length == 0) { throw new Error("Can't compose zero expressions"); } - let node: Expr | null = null; - const instantiation: ExpressionInput[] = []; - const transformer = astMapper(() => ({ - parameter: (st) => { - // All parameters are named ?, increase the index to avoid collisions with parameters we've already added. - const originalIndex = Number(st.name.substring(1)); - const newIndex = instantiation.length + originalIndex; - - return assignChanged(st, { name: `?${newIndex}` }); - } - })); - - for (const element of terms) { - if (node == null) { - node = element.expression.node; - } else { - const transformed = transformer.expr(element.expression.node)!; - - node = { - type: 'binary', - op, - left: node, - right: transformed, - _location: expandNodeLocations([node, transformed]) - }; - } + const [first, ...rest] = terms; + const locations = first.expression.locations; + let inner = first.expression.node; + for (const additional of rest) { + inner = { type: 'binary', operator, left: inner, right: additional.expression.node }; + } - instantiation.push(...element.expression.instantiationValues()); + const location = expandNodeLocations(terms.map((e) => e.expression.location)); + if (location) { + locations.sourceForNode.set(inner, location); } - const toSqlite = new PostgresToSqlite( - this.originalText, - { - report() { - // We don't need to re-report errors when we shuffle expressions around, the mapper would have already reported - // these issues on the first round. - } - }, - instantiation - ); - toSqlite.addExpression(node!); - return new SingleDependencyExpression(new SyncExpression(toSqlite.sql, node!, toSqlite.inputs)); + return new SingleDependencyExpression(new SyncExpression(inner, locations)); } } diff --git a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts index cd22b61d5..407188853 100644 --- a/packages/sync-rules/src/compiler/ir_to_sync_plan.ts +++ b/packages/sync-rules/src/compiler/ir_to_sync_plan.ts @@ -1,9 +1,11 @@ import * as plan from '../sync_plan/plan.js'; +import { SqlExpression } from '../sync_plan/expression.js'; import * as resolver from './bucket_resolver.js'; import { CompiledStreamQueries } from './compiler.js'; import { Equality, HashMap, StableHasher, unorderedEquality } from './equality.js'; -import { ColumnInRow, SyncExpression } from './expression.js'; +import { ColumnInRow, ExpressionInput, SyncExpression } from './expression.js'; import * as rows from './rows.js'; +import { MapSourceVisitor, visitExpr } from '../sync_plan/expression_visitor.js'; export class CompilerModelToSyncPlan { private static readonly evaluatorHash: Equality = unorderedEquality({ @@ -121,19 +123,16 @@ export class CompilerModelToSyncPlan { }); } - private translateExpression(expression: SyncExpression): plan.SqlExpression { - return { - sql: expression.sql, - values: expression.instantiation.map((e) => { - const value = e.value; - - if (value instanceof ColumnInRow) { - return { column: value.column } satisfies plan.ColumnSqlParameterValue; - } else { - return { request: value.source } satisfies plan.RequestSqlParameterValue; - } - }) as unknown[] as T[] - }; + private translateExpression(expression: SyncExpression): SqlExpression { + const mapper = new MapSourceVisitor((value) => { + if (value instanceof ColumnInRow) { + return { column: value.column } satisfies plan.ColumnSqlParameterValue as unknown as T; + } else { + return { request: value.source } satisfies plan.RequestSqlParameterValue as unknown as T; + } + }); + + return visitExpr(mapper, expression.node, null); } private translateStreamResolver(value: resolver.StreamResolver): plan.StreamQuerier { diff --git a/packages/sync-rules/src/compiler/parser.ts b/packages/sync-rules/src/compiler/parser.ts index 05ed9e36d..be50ac884 100644 --- a/packages/sync-rules/src/compiler/parser.ts +++ b/packages/sync-rules/src/compiler/parser.ts @@ -1,19 +1,4 @@ -import { - assignChanged, - astMapper, - BinaryOperator, - Expr, - ExprCall, - ExprParameter, - ExprRef, - From, - nil, - NodeLocation, - PGNode, - SelectedColumn, - SelectFromStatement, - Statement -} from 'pgsql-ast-parser'; +import { Expr, ExprCall, ExprRef, From, nil, NodeLocation, PGNode, SelectedColumn, Statement } from 'pgsql-ast-parser'; import { PhysicalSourceResultSet, RequestTableValuedResultSet, @@ -21,7 +6,7 @@ import { SyntacticResultSetSource } from './table.js'; import { ColumnSource, ExpressionColumnSource, StarColumnSource } from './rows.js'; -import { ColumnInRow, ConnectionParameter, ExpressionInput, SyncExpression } from './expression.js'; +import { ColumnInRow, ExpressionInput, NodeLocations, SyncExpression } from './expression.js'; import { BaseTerm, EqualsClause, @@ -34,12 +19,12 @@ import { } from './filter.js'; import { expandNodeLocations } from '../errors.js'; import { cartesianProduct } from '../streams/utils.js'; -import { intrinsicContains, PostgresToSqlite } from './sqlite.js'; +import { PostgresToSqlite } from './sqlite.js'; import { SqlScope } from './scope.js'; import { ParsingErrorListener, SyncStreamsCompiler } from './compiler.js'; import { TablePattern } from '../TablePattern.js'; import { FilterConditionSimplifier } from './filter_simplifier.js'; -import { ConnectionParameterSource } from '../sync_plan/plan.js'; +import { SqlExpression } from '../sync_plan/expression.js'; /** * A parsed stream query in its canonical form. @@ -86,28 +71,75 @@ export interface StreamQueryParserOptions { originalText: string; errors: ParsingErrorListener; parentScope?: SqlScope; + locations: NodeLocations; } export class StreamQueryParser { readonly errors: ParsingErrorListener; private readonly compiler: SyncStreamsCompiler; - private readonly originalText: string; + readonly originalText: string; private readonly statementScope: SqlScope; // Note: This is not the same as SqlScope since some result sets are inlined from CTEs or subqueries. These are not in // scope, but we still add them here to correctly track dependencies. private readonly resultSets = new Map(); private readonly resultColumns: ColumnSource[] = []; - private where: Expr[] = []; + private where: SqlExpression[] = []; /** The result set for which rows are synced. Set when analyzing result columns. */ private primaryResultSet?: PhysicalSourceResultSet; private syntheticSubqueryCounter: number = 0; + private nodeLocations: NodeLocations; + private exprParser: PostgresToSqlite; constructor(options: StreamQueryParserOptions) { this.compiler = options.compiler; this.originalText = options.originalText; this.errors = options.errors; this.statementScope = new SqlScope({ parent: options.parentScope }); + this.nodeLocations = options.locations; + + this.exprParser = new PostgresToSqlite({ + originalText: this.originalText, + errors: this.errors, + locations: this.nodeLocations, + resolveTableName: this.resolveTableName.bind(this), + generateTableAlias: () => { + const counter = this.syntheticSubqueryCounter++; + return `synthetic:${counter}`; + }, + joinSubqueryExpression: (expr) => { + // Independently analyze the inner query. + const parseInner = new StreamQueryParser({ + compiler: this.compiler, + originalText: this.originalText, + errors: this.errors, + parentScope: this.statementScope, + locations: this.nodeLocations + }); + let success = parseInner.processAst(expr, { forSubquery: true }); + if (!success) { + return null; + } + + let resultColumn: Expr | null = null; + if (expr.columns?.length == 1) { + const column = expr.columns[0].expr; + // The result of a subquery must be a scalar expression + if (!(column.type == 'ref' && column.name == '*')) { + resultColumn = column; + } + } + + if (resultColumn == null) { + // TODO: We could reasonably support syntax of the form (a, b) IN (SELECT a, b FROM ...) by desugaring that + // into multiple equals operators? The rest of the compiler should already be able to handle that. + this.errors.report('Must return a single expression column', expr); + return null; + } + + return { filters: parseInner.where, output: parseInner.parseExpression(resultColumn).node }; + } + }); } parse(stmt: Statement): ParsedStreamQuery | null { @@ -146,9 +178,6 @@ export class StreamQueryParser { return false; } - // Create scope and bind to statement - this.statementScope.bindingVisitor(node).statement(node); - node.from?.forEach((f) => this.processFrom(f)); if (node.where) { this.addAndTermToWhereClause(node.where); @@ -169,7 +198,7 @@ export class StreamQueryParser { } private addAndTermToWhereClause(expr: Expr) { - this.where.push(this.desugarSubqueries(expr)); + this.where.push(this.parseExpression(expr).node); } private processFrom(from: From) { @@ -224,12 +253,12 @@ export class StreamQueryParser { const resolvedArguments: RequestExpression[] = []; for (const argument of call.args) { - const parsed = this.mustBeSingleDependency(this.parseExpression(argument, true)); + const parsed = this.mustBeSingleDependency(this.parseExpression(argument)); if (parsed.resultSet != null) { this.errors.report( 'Parameters to table-valued functions may not reference other tables', - parsed.expression.node + parsed.expression.location ); } } @@ -262,9 +291,9 @@ export class StreamQueryParser { this.resultColumns.push(StarColumnSource.instance); } else { - const expr = this.parseExpression(column.expr, true); + const expr = this.parseExpression(column.expr); - for (const dependency of expr.instantiationValues()) { + for (const dependency of expr.instantiation) { if (dependency instanceof ColumnInRow) { selectsFrom(dependency.resultSet, dependency.syntacticOrigin); } else { @@ -295,26 +324,17 @@ export class StreamQueryParser { } } - private parseExpression(source: Expr, desugar: boolean): SyncExpression { - if (desugar) { - source = this.desugarSubqueries(source); - } - - return trackDependencies(this, source); + private parseExpression(source: Expr): SyncExpression { + return this.exprParser.translateExpression(source); } resolveTableName(node: ExprRef, name: string | nil): SourceResultSet | null { - const scope = SqlScope.readBoundScope(node); - if (scope == null) { - throw new Error('internal: Tried to resolve reference that has not been attached to a scope'); - } - if (name == null) { // For unqualified references, there must be a single table in scope. We don't allow unqualified references if // there are multiple tables because we don't know which column is available in which table with certainty (and // don't want to re-compile sync streams on schema changes). So, we just refuse to resolve those ambigious // references. - const resultSets = scope.resultSets; + const resultSets = this.statementScope.resultSets; if (resultSets.length == 1) { return this.resultSets.get(resultSets[0])!; } else { @@ -322,7 +342,7 @@ export class StreamQueryParser { return null; } } else { - const result = scope.resolveResultSetForReference(name); + const result = this.statementScope.resolveResultSetForReference(name); if (result == null) { this.errors.report(`Table '${name}' has not been added in a FROM clause here.`, node); return null; @@ -352,10 +372,10 @@ export class StreamQueryParser { private compileFilterClause(): Or { const andTerms: PendingFilterExpression[] = []; for (const expr of this.where) { - andTerms.push(this.extractBooleanOperators(this.desugarSubqueries(expr))); + andTerms.push(this.extractBooleanOperators(expr)); } - const pendingDnf = toDisjunctiveNormalForm({ type: 'and', inner: andTerms }); + const pendingDnf = toDisjunctiveNormalForm({ type: 'and', inner: andTerms }, this.nodeLocations); // Within the DNF, each base expression (that is, anything not an OR or AND) is either: // @@ -386,24 +406,17 @@ export class StreamQueryParser { private mapBaseExpression(pending: PendingBaseTerm): BaseTerm { if (pending.inner.type == 'binary') { - if (pending.inner.op == '=') { + if (pending.inner.operator == '=') { // The expression is of the form A = B. This introduces a parameter, allow A and B to reference different // result sets. - const left = this.parseExpression(pending.inner.left, false); - const right = this.parseExpression(pending.inner.right, false); + const left = new SyncExpression(pending.inner.left, this.nodeLocations); + const right = new SyncExpression(pending.inner.right, this.nodeLocations); return new EqualsClause(this.mustBeSingleDependency(left), this.mustBeSingleDependency(right)); } } - return this.mustBeSingleDependency(this.parseExpression(pending.inner, false)); - } - - toSyncExpression(source: Expr, instantiation: ExpressionInput[]): SyncExpression { - const toSqlite = new PostgresToSqlite(this.originalText, this.errors, instantiation); - toSqlite.addExpression(source); - - return new SyncExpression(toSqlite.sql, source, toSqlite.inputs); + return this.mustBeSingleDependency(new SyncExpression(pending.inner, this.nodeLocations)); } /** @@ -415,7 +428,7 @@ export class StreamQueryParser { let referencingConnection: PGNode | null = null; let hadError = false; - for (const dependency of inner.instantiationValues()) { + for (const dependency of inner.instantiation) { if (dependency instanceof ColumnInRow) { if (referencingConnection != null) { this.errors.report( @@ -448,235 +461,33 @@ export class StreamQueryParser { if (hadError) { // Return a bogus expression to keep going / potentially collect more errors. - return new SingleDependencyExpression( - new SyncExpression( - 'NULL', - { - type: 'null' - }, - [] - ) - ); + const value: SqlExpression = { type: 'lit_null' }; + this.nodeLocations.sourceForNode.set(value, inner.location); + return new SingleDependencyExpression(new SyncExpression(value, this.nodeLocations)); } else { return new SingleDependencyExpression(inner); } } - private extractBooleanOperators(source: Expr): PendingFilterExpression { + private extractBooleanOperators(source: SqlExpression): PendingFilterExpression { if (source.type == 'binary') { - if (source.op == 'AND') { + if (source.operator == 'and') { return { type: 'and', inner: [this.extractBooleanOperators(source.left), this.extractBooleanOperators(source.right)] }; - } else if (source.op == 'OR') { + } else if (source.operator == 'or') { return { type: 'or', inner: [this.extractBooleanOperators(source.left), this.extractBooleanOperators(source.right)] }; } - } else if (source.type == 'unary' && source.op == 'NOT') { + } else if (source.type == 'unary' && source.operator == 'not') { return { type: 'not', inner: this.extractBooleanOperators(source.operand) }; } return { type: 'base', inner: source }; } - - /** - * Desugars valid forms of subqueries in the source expression. - * - * In particular, this lowers `x IN (SELECT ...)` by adding the inner select statement as an `OUTER JOIN` and then - * replacing the `IN` operator with `x = joinedTable.value`. - */ - desugarSubqueries(source: Expr): Expr { - const mapper = astMapper((map) => { - // Desugar left IN ARRAY(...right) to "intrinsic:contains"(left, ...right). This will eventually get compiled to - // the SQLite expression LEFT IN (...right), using row-values syntax. - function desugarInValues(negated: boolean, left: Expr, right: Expr[]) { - const containsCall: Expr = { type: 'call', function: { name: intrinsicContains }, args: [left, ...right] }; - if (negated) { - return map.super().expr({ type: 'unary', op: 'NOT', operand: containsCall }); - } else { - return map.super().expr(containsCall); - } - } - - const desugarInSubquery = (negated: boolean, left: Expr, right: SelectFromStatement) => { - // Independently analyze the inner query. - const parseInner = new StreamQueryParser({ - compiler: this.compiler, - originalText: this.originalText, - errors: this.errors, - parentScope: this.statementScope - }); - let success = parseInner.processAst(right, { forSubquery: true }); - let resultColumn: Expr | null = null; - if (right.columns?.length == 1) { - const column = right.columns[0].expr; - // The result of a subquery must be a scalar expression - if (!(column.type == 'ref' && column.name == '*')) { - resultColumn = column; - } - } - - if (resultColumn == null) { - // TODO: We could reasonably support syntax of the form (a, b) IN (SELECT a, b FROM ...) by desugaring that - // into multiple equals operators? The rest of the compiler should already be able to handle that. - this.errors.report('Must return a single expression column', right); - success = false; - } - if (!success || resultColumn == null) { - return map.expr({ type: 'null', _location: right._location }); - } - - // Inline the subquery by adding all referenced tables to the main query, adding the filter and replacing - // `a IN (SELECT b FROM ...)` with `a = joined.b`. - parseInner.resultSets.forEach((v, k) => this.resultSets.set(k, v)); - let replacement: Expr = { type: 'binary', op: '=', left: left, right: resultColumn }; - if (parseInner.where != null) { - replacement = parseInner.where.reduce((prev, current) => { - return { - type: 'binary', - left: prev, - right: current, - op: 'AND' - } satisfies Expr; - }, replacement); - } - - if (negated) { - replacement = { type: 'unary', op: 'NOT', operand: replacement }; - } - return replacement; - }; - - // Desugar left IN right, where right is a scalar expression. This is not valid SQL, but in PowerSync we interpret - // that as `left IN (SELECT value FROM json_each(right))`. - const desugarInScalar = (negated: boolean, left: Expr, right: Expr) => { - const counter = this.syntheticSubqueryCounter++; - const name = `synthetic:${counter}`; - return desugarInSubquery(negated, left, { - type: 'select', - columns: [{ expr: { type: 'ref', name: 'value', table: { name } } }], - from: [{ type: 'call', function: { name: 'json_each' }, args: [right], alias: { name } }] - }); - }; - - return { - binary: (expr) => { - if (expr.op == 'IN' || expr.op == 'NOT IN') { - const right = expr.right; - const negated = expr.op == 'NOT IN'; - - if (right.type == 'select') { - return desugarInSubquery(negated, expr.left, right); - } else if (right.type == 'array') { - return desugarInValues(negated, expr.left, right.expressions); - } else if (right.type == 'call' && right.function.name.toLowerCase() == 'row') { - return desugarInValues(negated, expr.left, right.args); - } else { - return desugarInScalar(negated, expr.left, right); - } - } - - return map.super().binary(expr); - } - }; - }); - - return mapper.expr(source)!; - } -} - -function trackDependencies(parser: StreamQueryParser, source: Expr): SyncExpression { - const instantiation: ExpressionInput[] = []; - const mapper = astMapper((map) => { - function createParameter(node: PGNode): ExprParameter { - return { - type: 'parameter', - name: `?${instantiation.length}`, - _location: node._location - }; - } - - function replaceWithParameter(node: PGNode) { - return map.super().parameter(createParameter(node)); - } - - return { - ref: (val) => { - const resultSet = parser.resolveTableName(val, val.table?.name); - if (val.name == '*') { - parser.errors.report('* columns are not supported here', val); - } - - if (resultSet == null || val.name == '*') { - // resolveTableName will have logged an error, so transform with a bogus value to keep going. - return { type: 'null', _location: val._location }; - } - - instantiation.push(new ColumnInRow(val, resultSet, val.name)); - return replaceWithParameter(val); - }, - call: (val) => { - const schemaName = val.function.schema; - const source: ConnectionParameterSource | null = - schemaName === 'auth' || schemaName === 'subscription' || schemaName === 'connection' ? schemaName : null; - if (!source) { - return map.super().call(val); - } - - const parameter = new ConnectionParameter(val, source); - instantiation.push(parameter); - const replacement = createParameter(val); - - switch (val.function.name.toLowerCase()) { - case 'parameters': - break; - case 'parameter': - // Desugar .param(x) into .parameters() ->> '$.' || x - if (val.args.length == 1) { - return map.super().binary({ - type: 'binary', - left: replacement, - op: '->>' as BinaryOperator, - right: { - type: 'binary', - left: { type: 'string', value: '$.' }, - op: '||', - right: val.args[0] - }, - _location: val._location - }); - } else { - parser.errors.report('Expected a single argument here', val.function); - } - case 'user_id': - if (source == 'auth') { - // Desugar auth.user_id() into auth.parameters() ->> '$.sub' - return map.super().binary({ - type: 'binary', - left: replacement, - op: '->>' as BinaryOperator, - right: { type: 'string', value: '$.sub' }, - _location: val._location - }); - } else { - parser.errors.report('.user_id() is only available on auth schema', val.function); - } - break; - default: - parser.errors.report('Unknown request function', val.function); - } - - // Return the entire JSON object - return map.super().parameter(replacement); - } - }; - }); - - const transformed = mapper.expr(source)!; - return parser.toSyncExpression(transformed, instantiation); } /** @@ -691,11 +502,11 @@ type PendingFilterExpression = | PendingOr | PendingBaseTerm; -type PendingBaseTerm = { type: 'base'; inner: Expr }; +type PendingBaseTerm = { type: 'base'; inner: SqlExpression }; type PendingOr = { type: 'or'; inner: PendingFilterExpression[] }; -function toDisjunctiveNormalForm(source: PendingFilterExpression): PendingOr { - const prepared = prepareToDNF(source); +function toDisjunctiveNormalForm(source: PendingFilterExpression, locations: NodeLocations): PendingOr { + const prepared = prepareToDNF(source, locations); switch (prepared.type) { case 'or': return { @@ -715,7 +526,7 @@ function toDisjunctiveNormalForm(source: PendingFilterExpression): PendingOr { } } -function prepareToDNF(expr: PendingFilterExpression): PendingFilterExpression { +function prepareToDNF(expr: PendingFilterExpression, locations: NodeLocations): PendingFilterExpression { switch (expr.type) { case 'not': { // Push NOT downwards, depending on the inner term. @@ -725,19 +536,30 @@ function prepareToDNF(expr: PendingFilterExpression): PendingFilterExpression { return inner.inner; // Double negation, !x => x case 'and': // !(a AND b) => (!a) OR (!b) - return prepareToDNF({ type: 'or', inner: inner.inner.map((e) => prepareToDNF({ type: 'not', inner: e })) }); + return prepareToDNF( + { + type: 'or', + inner: inner.inner.map((e) => prepareToDNF({ type: 'not', inner: e }, locations)) + }, + locations + ); case 'or': // !(a OR b) => (!a) AND (!b) - return prepareToDNF({ type: 'and', inner: inner.inner.map((e) => prepareToDNF({ type: 'not', inner: e })) }); + return prepareToDNF( + { + type: 'and', + inner: inner.inner.map((e) => prepareToDNF({ type: 'not', inner: e }, locations)) + }, + locations + ); case 'base': - return { - type: 'base', - inner: { - type: 'unary', - op: 'NOT', - operand: inner.inner - } + const mappedInner: SqlExpression = { + type: 'unary', + operator: 'not', + operand: inner.inner }; + locations.sourceForNode.set(mappedInner, locations.locationFor(inner.inner)); + return { type: 'base', inner: mappedInner }; } } case 'and': { @@ -745,7 +567,7 @@ function prepareToDNF(expr: PendingFilterExpression): PendingFilterExpression { const orTerms: PendingOr[] = []; for (const originalTerm of expr.inner) { - const normalized = prepareToDNF(originalTerm); + const normalized = prepareToDNF(originalTerm, locations); if (normalized.type == 'and') { // Normalized and will only have base terms as children baseFactors.push(...(normalized.inner as PendingBaseTerm[])); @@ -769,7 +591,7 @@ function prepareToDNF(expr: PendingFilterExpression): PendingFilterExpression { // Then, combine those with the inner AND to turn `A & (B | C) & D` into `(B & A & D) | (C & A & D)`. const finalFactors: PendingFilterExpression[] = []; for (const distributedTerms of multiplied) { - finalFactors.push(prepareToDNF({ type: 'and', inner: [...distributedTerms, ...baseFactors] })); + finalFactors.push(prepareToDNF({ type: 'and', inner: [...distributedTerms, ...baseFactors] }, locations)); } return { type: 'or', inner: finalFactors }; } @@ -778,7 +600,7 @@ function prepareToDNF(expr: PendingFilterExpression): PendingFilterExpression { // if possible. const expanded: PendingFilterExpression[] = []; for (const term of expr.inner) { - const normalized = prepareToDNF(term); + const normalized = prepareToDNF(term, locations); if (normalized.type == 'or') { expanded.push(...normalized.inner); } else { diff --git a/packages/sync-rules/src/compiler/querier_graph.ts b/packages/sync-rules/src/compiler/querier_graph.ts index b9f49a197..369aabec5 100644 --- a/packages/sync-rules/src/compiler/querier_graph.ts +++ b/packages/sync-rules/src/compiler/querier_graph.ts @@ -157,7 +157,7 @@ class PendingQuerierPath { if (remaining.resultSet != null) { this.errors.report( 'This filter is unrelated to the request or the table being synced, and not supported.', - remaining.expression.node! + remaining.expression.location ); } else { requestConditions.push(new RequestExpression(remaining)); diff --git a/packages/sync-rules/src/compiler/scope.ts b/packages/sync-rules/src/compiler/scope.ts index c2ace2018..3ed310b36 100644 --- a/packages/sync-rules/src/compiler/scope.ts +++ b/packages/sync-rules/src/compiler/scope.ts @@ -1,4 +1,3 @@ -import { astVisitor, ExprRef, PGNode } from 'pgsql-ast-parser'; import { SyntacticResultSetSource } from './table.js'; import { ParsingErrorListener } from './compiler.js'; @@ -32,45 +31,4 @@ export class SqlScope { resolveResultSetForReference(name: string): SyntacticResultSetSource | undefined { return this.nameToResultSet.get(name.toLowerCase()) ?? this.parent?.resolveResultSetForReference(name); } - - /** - * Returns a visitor binding references in AST nodes it runs on to this scope. - * - * This is used to preserve scopes when smuggling references out of subqueries. The parser transforms subqueries from - * `WHERE outer = (SELECT inner FROM a ...)` to `JOIN a ON ... AND outer = inner`. In the transformed form, we need - * to preserve original scopes for `outer` and `inner`. - */ - bindingVisitor(root: PGNode) { - return astVisitor((v) => ({ - ref: (expr) => { - (expr as any)[boundScope] = this; - }, - select(stmt) { - if (stmt !== root) { - // Return and don't visit children (since those have their own scope). - return; - } else { - v.super().select(stmt); - } - }, - statement(stmt) { - if (stmt !== root) { - // Return and don't visit children (since those have their own scope). - return; - } else { - v.super().statement(stmt); - } - } - })); - } - - /** - * If the given node was part of an AST previously passed to {@link bindTo}, returns the - * scope attached to it. - */ - static readBoundScope(node: ExprRef): SqlScope | undefined { - return (node as any)[boundScope]; - } } - -const boundScope = Symbol('SqlScope.boundScope'); diff --git a/packages/sync-rules/src/compiler/sqlite.ts b/packages/sync-rules/src/compiler/sqlite.ts index 5e11ad729..b0a268ff6 100644 --- a/packages/sync-rules/src/compiler/sqlite.ts +++ b/packages/sync-rules/src/compiler/sqlite.ts @@ -1,169 +1,157 @@ -import { BinaryOperator, Expr, ExprCall, UnaryOperator } from 'pgsql-ast-parser'; -import { ParsingErrorListener } from './compiler.js'; +import { + BinaryOperator, + Expr, + ExprBinary, + ExprCall, + ExprRef, + nil, + PGNode, + SelectFromStatement +} from 'pgsql-ast-parser'; import { CAST_TYPES } from '../sql_functions.js'; -import { ExpressionInput, ExpressionInputWithSpan } from './expression.js'; +import { ColumnInRow, ConnectionParameter, ExpressionInput, NodeLocations, SyncExpression } from './expression.js'; +import { + BetweenExpression, + LiteralExpression, + SqlExpression, + supportedFunctions, + BinaryOperator as SupportedBinaryOperator +} from '../sync_plan/expression.js'; +import { ConnectionParameterSource } from '../sync_plan/plan.js'; +import { ParsingErrorListener } from './compiler.js'; +import { SourceResultSet } from './table.js'; + +export interface ResolvedSubqueryExpression { + filters: SqlExpression[]; + output: SqlExpression; +} -export const intrinsicContains = 'intrinsic:contains'; +export interface PostgresToSqliteOptions { + readonly originalText: string; + readonly errors: ParsingErrorListener; + readonly locations: NodeLocations; + + /** + * Attempt to resolve a table name in scope, returning the resolved result set. + * + * Should report an error if resolving the table failed, using `node` as the source location for the error. + */ + resolveTableName(node: ExprRef, name: string | nil): SourceResultSet | null; + + /** + * Generates a table alias for synthetic subqueries like those generated to desugar `IN` expressions to `json_each` + * subqueries. + */ + generateTableAlias(): string; + + /** + * Turns the given subquery into a join added to the main `FROM` section. + * + * Returns the parsed subquery expression, or null if resolving the subquery failed. In that case, this method should + * report an error. + */ + joinSubqueryExpression(expr: SelectFromStatement): ResolvedSubqueryExpression | null; +} /** - * Utility for translating Postgres expressions to SQLite expressions. + * Validates and lowers a Postgres expression into a scalar SQL expression. * * Unsupported Postgres features are reported as errors. */ export class PostgresToSqlite { - sql = ''; - inputs: ExpressionInputWithSpan[] = []; - - private needsSpace = false; - - constructor( - private readonly originalSource: string, - private readonly errors: ParsingErrorListener, - private readonly parameters: ExpressionInput[] - ) {} + constructor(private readonly options: PostgresToSqliteOptions) {} - private space() { - this.sql += ' '; + translateExpression(source: Expr): SyncExpression { + return new SyncExpression(this.translateNodeWithLocation(source), this.options.locations); } - private addLexeme(text: string, options?: { spaceLeft?: boolean; spaceRight?: boolean }): number { - const spaceLeft = options?.spaceLeft ?? true; - const spaceRight = options?.spaceRight ?? true; - - if (this.needsSpace && spaceLeft) { - this.space(); - } - - const startOffset = this.sql.length; - this.sql += text; - this.needsSpace = spaceRight; - return startOffset; + private translateNodeWithLocation(expr: Expr): SqlExpression { + const translated = this.translateToNode(expr); + this.options.locations.sourceForNode.set(translated, expr); + return translated; } - private identifier(name: string) { - this.addLexeme(`"${name.replaceAll('"', '""')}"`); - } - - private string(name: string) { - this.addLexeme(`'${name.replaceAll("'", "''")}'`); - } - - private bogusExpression() { - this.addLexeme('NULL'); - } - - private commaSeparated(exprList: Iterable) { - let first = true; - for (const expr of exprList) { - if (!first) { - this.addLexeme(',', { spaceLeft: false }); - } - - this.addExpression(expr); - first = false; - } - } - - private intrinsicContains(call: ExprCall, outerPrecedence: Precedence | 0) { - const [left, ...right] = call.args; - if (right.length == 0) { - this.addLexeme('FALSE'); - return; - } - - this.maybeParenthesis(outerPrecedence, Precedence.equals, () => { - this.addExpression(left, Precedence.equals); - this.addLexeme('IN'); - this.parenthesis(() => this.commaSeparated(right)); - }); - } - - private parenthesis(inner: () => void) { - this.addLexeme('(', { spaceRight: false }); - inner(); - this.addLexeme(')', { spaceLeft: false }); - } - - private maybeParenthesis(outerPrecedence: Precedence | 0, innerPrecedence: Precedence, inner: () => void) { - if (outerPrecedence > innerPrecedence) { - this.parenthesis(inner); - } else { - inner(); - } - } - - addExpression(expr: Expr, outerPrecedence: Precedence | 0 = 0) { - // export type Expr = | ExprExtract | ExprMember ; + private translateToNode(expr: Expr): SqlExpression { switch (expr.type) { case 'null': - this.addLexeme('NULL'); - break; + return { type: 'lit_null' }; case 'boolean': - this.addLexeme(expr.value ? 'TRUE' : 'FALSE'); - break; + return { type: 'lit_int', base10: expr.value ? '1' : '0' }; case 'string': - this.string(expr.value); - break; - case 'numeric': + return { type: 'lit_string', value: expr.value }; + case 'numeric': { + return { type: 'lit_double', value: expr.value }; + } case 'integer': { - // JavaScript does not have a number type that can represent SQLite values, so we try to reuse the source. + // JavaScript does not have a number type that can represent SQLite ints, so we try to reuse the source. if (expr._location) { - this.addLexeme(this.originalSource.substring(expr._location.start, expr._location.end)); + return { + type: 'lit_int', + base10: this.options.originalText.substring(expr._location.start, expr._location.end) + }; } else { - this.addLexeme(expr.value.toString()); + return { type: 'lit_int', base10: expr.value.toString() }; } - break; } - case 'ref': - this.bogusExpression(); - this.errors.report('Internal error: Dependency should have been extracted', expr); - break; - case 'parameter': - // The name is going to be ? - const index = Number(expr.name.substring(1)); - const value = this.parameters[index - 1]; - if (value == null) { - throw new Error('Internal error: No value given for parameter'); + case 'ref': { + const resultSet = this.options.resolveTableName(expr, expr.table?.name); + if (resultSet == null) { + // resolveTableName will have logged an error, transform with a bogus value to keep going. + return { type: 'lit_null' }; } - const start = this.addLexeme('?'); - this.inputs.push(new ExpressionInputWithSpan(value, start, 1)); - break; + + if (expr.name == '*') { + return this.invalidExpression(expr, '* columns are not supported here'); + } + + const instantiation = new ColumnInRow(expr, resultSet, expr.name); + return { + type: 'data', + source: instantiation + }; + } + case 'parameter': + return this.invalidExpression( + expr, + 'SQL parameters are not allowed. Use parameter functions instead: https://docs.powersync.com/usage/sync-streams#accessing-parameters' + ); case 'substring': { - const args = [expr.value, expr.from ?? { type: 'numeric', value: 1 }]; - if (expr.for != null) { - args.push(expr.for); + const mappedArgs = [this.translateNodeWithLocation(expr.value)]; + if (expr.from) { + mappedArgs.push(this.translateNodeWithLocation(expr.from)); + } else { + mappedArgs.push({ type: 'lit_int', base10: '1' }); } - this.addExpression({ - type: 'call', - function: { name: 'substr' }, - args - }); - break; + if (expr.for) { + mappedArgs.push(this.translateNodeWithLocation(expr.for)); + } + + return { type: 'function', function: 'substr', parameters: mappedArgs }; } case 'call': { - if (expr.function.name == intrinsicContains) { - // Calls to this function should be lowered to a IN (x, y, z). We can't represent that in Postgres AST. - return this.intrinsicContains(expr, outerPrecedence); - } + const schemaName = expr.function.schema; + const source: ConnectionParameterSource | null = + schemaName === 'auth' || schemaName === 'subscription' || schemaName === 'connection' ? schemaName : null; - if (expr.function.schema) { - this.errors.report('Invalid schema in function name', expr.function); - return this.bogusExpression(); + if (schemaName) { + if (source) { + return this.translateRequestParameter(source, expr); + } else { + return this.invalidExpression(expr.function, 'Invalid schema in function name'); + } } + if (expr.distinct != null || expr.orderBy != null || expr.filter != null || expr.over != null) { - this.errors.report('DISTINCT, ORDER BY, FILTER and OVER clauses are not supported', expr.function); - return this.bogusExpression(); + return this.invalidExpression(expr.function, 'DISTINCT, ORDER BY, FILTER and OVER clauses are not supported'); } const forbiddenReason = forbiddenFunctions[expr.function.name]; if (forbiddenReason) { - this.errors.report(`Forbidden call: ${forbiddenReason}`, expr.function); - return this.bogusExpression(); + return this.invalidExpression(expr.function, `Forbidden call: ${forbiddenReason}`); } let allowedArgs = supportedFunctions[expr.function.name]; if (allowedArgs == null) { - this.errors.report('Unknown function', expr.function); - return this.bogusExpression(); + return this.invalidExpression(expr.function, 'Unknown function'); } else { if (typeof allowedArgs == 'number') { allowedArgs = { min: allowedArgs, max: allowedArgs }; @@ -171,101 +159,129 @@ export class PostgresToSqlite { const actualArgs = expr.args.length; if (actualArgs < allowedArgs.min) { - this.errors.report(`Expected at least ${allowedArgs.min} arguments`, expr); + this.options.errors.report(`Expected at least ${allowedArgs.min} arguments`, expr); } else if (allowedArgs.max && actualArgs > allowedArgs.max) { - this.errors.report(`Expected at most ${allowedArgs.max} arguments`, expr); + this.options.errors.report(`Expected at most ${allowedArgs.max} arguments`, expr); } else if (allowedArgs.mustBeEven && actualArgs % 2 == 1) { - this.errors.report(`Expected an even amount of arguments`, expr); + this.options.errors.report(`Expected an even amount of arguments`, expr); } else if (allowedArgs.mustBeOdd && actualArgs % 2 == 0) { - this.errors.report(`Expected an odd amount of arguments`, expr); + this.options.errors.report(`Expected an odd amount of arguments`, expr); } } - this.identifier(expr.function.name); - this.addLexeme('(', { spaceLeft: false, spaceRight: false }); - this.commaSeparated(expr.args); - this.addLexeme(')', { spaceLeft: false }); - break; + return { + type: 'function', + function: expr.function.name, + parameters: expr.args.map((a) => this.translateNodeWithLocation(a)) + }; } case 'binary': { - const precedence = supportedBinaryOperators[expr.op]; - if (precedence == null) { - this.bogusExpression(); - this.errors.report('Unsupported binary operator', expr); + if (expr.op === 'IN' || expr.op === 'NOT IN') { + return this.translateInExpression(expr); + } + + const left = this.translateNodeWithLocation(expr.left); + const right = this.translateNodeWithLocation(expr.right); + if (expr.op === 'LIKE') { + return { type: 'function', function: 'like', parameters: [left, right] }; + } else if (expr.op === 'NOT LIKE') { + return { + type: 'unary', + operator: 'not', + operand: { type: 'function', function: 'like', parameters: [left, right] } + }; + } else if (expr.op === '!=') { + return { + type: 'unary', + operator: 'not', + operand: { type: 'binary', left, right, operator: '=' } + }; + } + + const supported = supportedBinaryOperators[expr.op]; + if (supported == null) { + return this.invalidExpression(expr, 'Unsupported binary operator'); } else { - this.maybeParenthesis(outerPrecedence, precedence, () => { - this.addExpression(expr.left, precedence); - this.addLexeme(expr.op); - this.addExpression(expr.right, precedence); - }); + return { type: 'binary', left, right, operator: supported }; } - break; } case 'unary': { - const [isSuffix, precedence] = supportedUnaryOperators[expr.op]; - - this.maybeParenthesis(outerPrecedence, precedence, () => { - if (!isSuffix) this.addLexeme(expr.op); - this.addExpression(expr.operand, precedence); - if (isSuffix) this.addLexeme(expr.op); - }); - break; + let not = false; + let rightHandSideOfIs: SqlExpression; + + switch (expr.op) { + case '+': + case '-': + return { type: 'unary', operator: expr.op, operand: this.translateNodeWithLocation(expr.operand) }; + case 'NOT': + return { type: 'unary', operator: 'not', operand: this.translateNodeWithLocation(expr.operand) }; + case 'IS NOT NULL': + not = true; + case 'IS NULL': // fallthrough + rightHandSideOfIs = { type: 'lit_null' }; + break; + case 'IS NOT TRUE': + not = true; + case 'IS TRUE': // fallthrough + rightHandSideOfIs = { type: 'lit_int', base10: '1' }; + break; + case 'IS NOT FALSE': // fallthrough + not = true; + case 'IS FALSE': + rightHandSideOfIs = { type: 'lit_int', base10: '0' }; + break; + } + + const mappedIs: SqlExpression = { + type: 'binary', + left: this.translateNodeWithLocation(expr.operand), + operator: 'is', + right: rightHandSideOfIs + }; + + return not ? { type: 'unary', operator: 'not', operand: mappedIs } : mappedIs; } case 'cast': { const to = (expr.to as any)?.name?.toLowerCase() as string | undefined; if (to == null || !CAST_TYPES.has(to)) { - this.errors.report('Invalid SQLite cast', expr.to); - return this.bogusExpression(); + return this.invalidExpression(expr.to, 'Invalid SQLite cast'); } else { - this.addLexeme('CAST(', { spaceRight: false }); - this.addExpression(expr.operand); - this.addLexeme('AS'); - this.addLexeme(to); - this.addLexeme(')', { spaceLeft: false }); + return { type: 'cast', operand: this.translateNodeWithLocation(expr.operand), cast_as: to as any }; } - - break; } case 'ternary': { - this.maybeParenthesis(outerPrecedence, Precedence.equals, () => { - this.addExpression(expr.value, Precedence.equals); - this.addLexeme(expr.op); - this.addExpression(expr.lo, Precedence.equals); - this.addLexeme('AND'); - this.addExpression(expr.hi, Precedence.equals); - }); - break; + const between: BetweenExpression = { + type: 'between', + value: this.translateNodeWithLocation(expr.value), + low: this.translateNodeWithLocation(expr.lo), + high: this.translateNodeWithLocation(expr.hi) + }; + + return expr.op === 'BETWEEN' ? between : { type: 'unary', operator: 'not', operand: between }; } case 'case': { - this.addLexeme('CASE'); - if (expr.value) { - this.addExpression(expr.value); - } - for (const when of expr.whens) { - this.addLexeme('WHEN'); - this.addExpression(when.when); - this.addLexeme('THEN'); - this.addExpression(when.value); - } - - if (expr.else) { - this.addLexeme('ELSE'); - this.addExpression(expr.else); - } - this.addLexeme('END'); - break; + return { + type: 'case_when', + operand: expr.value ? this.translateNodeWithLocation(expr.value) : undefined, + whens: expr.whens.map((when) => ({ + when: this.translateNodeWithLocation(when.when), + then: this.translateNodeWithLocation(when.value) + })), + else: expr.else ? this.translateNodeWithLocation(expr.else) : undefined + }; } case 'member': { - this.maybeParenthesis(outerPrecedence, Precedence.concat, () => { - this.addExpression(expr.operand, Precedence.concat); - this.addLexeme(expr.op); - if (typeof expr.member == 'number') { - this.addExpression({ type: 'integer', value: expr.member }); - } else { - this.addExpression({ type: 'string', value: expr.member }); - } - }); - break; + const operand = this.translateNodeWithLocation(expr.operand); + return { + type: 'function', + function: expr.op, + parameters: [ + operand, + typeof expr.member == 'number' + ? { type: 'lit_int', base10: expr.member.toString() } + : { type: 'lit_string', value: expr.member } + ] + }; } case 'select': case 'union': @@ -273,135 +289,146 @@ export class PostgresToSqlite { case 'with': case 'with recursive': // Should have been desugared. - this.bogusExpression(); - this.errors.report('Invalid position for subqueries. Subqueries are only supported in WHERE clauses.', expr); - break; + return this.invalidExpression( + expr, + 'Invalid position for subqueries. Subqueries are only supported in WHERE clauses.' + ); default: - expr.type; - this.bogusExpression(); - this.errors.report('This expression is not supported by PowerSync', expr); + return this.invalidExpression(expr, 'This expression is not supported by PowerSync'); } } -} -enum Precedence { - or = 1, - and = 2, - not = 3, - equals = 4, - comparison = 5, - binary = 6, - addition = 7, - multiplication = 8, - concat = 9, - collate = 10, - unary = 11 -} + private invalidExpression(source: PGNode, message: string): LiteralExpression { + this.options.errors.report(message, source); + return { type: 'lit_null' }; + } -type ArgumentCount = number | { min: number; max?: number; mustBeEven?: boolean; mustBeOdd?: boolean }; - -const supportedFunctions: Record = { - // https://sqlite.org/lang_corefunc.html#list_of_core_functions - abs: 1, - char: { min: 0 }, - coalesce: { min: 2 }, - concat: { min: 1 }, - concat_ws: { min: 2 }, - format: { min: 1 }, - glob: 2, - hex: 1, - ifnull: { min: 2 }, - if: { min: 2 }, - iif: { min: 2 }, - instr: 2, - length: 1, - like: { min: 2, max: 3 }, - likelihood: 2, - likely: 1, - lower: 1, - ltrim: { min: 1, max: 2 }, - max: { min: 2 }, - min: { min: 2 }, - nullif: 2, - octet_length: 1, - printf: { min: 1 }, - quote: 1, - replace: 3, - round: { min: 1, max: 2 }, - rtrim: { min: 1, max: 2 }, - sign: 1, - substr: { min: 2, max: 3 }, - substring: { min: 2, max: 3 }, - trim: { min: 1, max: 2 }, - typeof: 1, - unhex: { min: 1, max: 2 }, - unicode: 1, - unistr: 1, - unistr_quote: 1, - unlikely: 1, - upper: 1, - zeroblob: 1, - // Scalar functions from https://sqlite.org/json1.html#overview - json: 1, - jsonb: 1, - json_array: { min: 0 }, - jsonb_array: { min: 0 }, - json_array_length: { min: 1, max: 2 }, - json_error_position: 1, - json_extract: { min: 2 }, - jsonb_extract: { min: 2 }, - json_insert: { min: 3, mustBeOdd: true }, - jsonb_insert: { min: 3, mustBeOdd: true }, - json_object: { min: 0, mustBeEven: true }, - jsonb_object: { min: 0, mustBeEven: true }, - json_patch: 2, - jsonb_patch: 2, - json_pretty: 1, - json_remove: { min: 2 }, - jsonb_remove: { min: 2 }, - json_replace: { min: 3, mustBeOdd: true }, - jsonb_replace: { min: 3, mustBeOdd: true }, - json_set: { min: 3, mustBeOdd: true }, - jsonb_set: { min: 3, mustBeOdd: true }, - json_type: { min: 1, max: 2 }, - json_valid: { min: 1, max: 2 }, - json_quote: { min: 1 } -}; + private translateInExpression(expr: ExprBinary): SqlExpression { + const negated = expr.op === 'NOT IN'; + const right = expr.right; -const supportedBinaryOperators: Partial> = { - OR: Precedence.or, - AND: Precedence.and, - '=': Precedence.equals, - '!=': Precedence.equals, - LIKE: Precedence.equals, - '<': Precedence.comparison, - '>': Precedence.comparison, - '<=': Precedence.comparison, - '>=': Precedence.comparison, - '&': Precedence.binary, - '|': Precedence.binary, - '<<': Precedence.binary, - '>>': Precedence.binary, - '+': Precedence.addition, - '-': Precedence.addition, - '*': Precedence.multiplication, - '/': Precedence.multiplication, - '%': Precedence.multiplication, - '||': Precedence.concat, - ['->' as BinaryOperator]: Precedence.concat, - ['->>' as BinaryOperator]: Precedence.concat -}; + if (right.type == 'select') { + return this.desugarInSubquery(negated, expr, right); + } else if (right.type == 'array') { + return this.desugarInValues(negated, expr.left, right.expressions); + } else if (right.type == 'call' && right.function.name.toLowerCase() == 'row') { + return this.desugarInValues(negated, expr.left, right.args); + } else { + return this.desugarInScalar(negated, expr, right); + } + } + + private desugarInValues(negated: boolean, left: Expr, right: Expr[]): SqlExpression { + const scalarIn: SqlExpression = { + type: 'scalar_in', + target: this.translateNodeWithLocation(left), + in: right.map((e) => this.translateNodeWithLocation(e)) + }; + return negated ? { type: 'unary', operator: 'not', operand: scalarIn } : scalarIn; + } + + private desugarInSubquery( + negated: boolean, + binary: ExprBinary, + right: SelectFromStatement + ): SqlExpression { + const left = binary.left; + const resolved = this.options.joinSubqueryExpression(right); + if (resolved == null) { + // An error would have been logged. + return { type: 'lit_null' }; + } + + let replacement: SqlExpression = { + type: 'binary', + operator: '=', + left: this.translateNodeWithLocation(left), + right: resolved.output + }; + this.options.locations.sourceForNode.set(replacement, binary); + + replacement = resolved.filters.reduce( + (left, right) => ({ type: 'binary', operator: 'and', left, right }), + replacement + ); + + if (negated) { + replacement = { type: 'unary', operator: 'not', operand: replacement }; + } + + return replacement; + } + + /** + * Desugar `$left IN $right`, where `$right` is a scalar expression. This is not valid SQL, but in PowerSync we + * interpret that as `left IN (SELECT value FROM json_each(right))`. + */ + private desugarInScalar(negated: boolean, binary: ExprBinary, right: Expr): SqlExpression { + const name = this.options.generateTableAlias(); + return this.desugarInSubquery(negated, binary, { + type: 'select', + columns: [{ expr: { type: 'ref', name: 'value', table: { name } } }], + from: [{ type: 'call', function: { name: 'json_each' }, args: [right], alias: { name } }] + }); + } + + private translateRequestParameter(source: ConnectionParameterSource, expr: ExprCall): SqlExpression { + const parameter = new ConnectionParameter(expr, source); + const replacement: SqlExpression = { + type: 'data', + source: parameter + }; + this.options.locations.sourceForNode.set(replacement, expr.function); + + switch (expr.function.name.toLowerCase()) { + case 'parameters': + return replacement; + case 'parameter': + // Desugar .param(x) into .parameters() ->> '$.' || x + if (expr.args.length == 1) { + return { + type: 'function', + function: '->>', + parameters: [replacement, this.translateNodeWithLocation(expr.args[0])] + }; + } else { + return this.invalidExpression(expr.function, 'Expected a single argument here'); + } + case 'user_id': + if (source == 'auth') { + // Desugar auth.user_id() into auth.parameters() ->> '$.sub' + return { + type: 'function', + function: '->>', + parameters: [replacement, { type: 'lit_string', value: '$.sub' }] + }; + } else { + return this.invalidExpression(expr.function, '.user_id() is only available on auth schema'); + } + default: + return this.invalidExpression(expr.function, 'Unknown request function'); + } + } +} -const supportedUnaryOperators: Record = { - NOT: [false, Precedence.not], - 'IS NULL': [true, Precedence.equals], - 'IS NOT NULL': [true, Precedence.equals], - 'IS FALSE': [true, Precedence.equals], - 'IS NOT FALSE': [true, Precedence.equals], - 'IS TRUE': [true, Precedence.equals], - 'IS NOT TRUE': [true, Precedence.equals], - '+': [false, Precedence.unary], - '-': [false, Precedence.unary] +const supportedBinaryOperators: Partial> = { + OR: 'or', + AND: 'and', + '=': '=', + '<': '<', + '>': '>', + '<=': '<=', + '>=': '>=', + '&': '&', + '|': '|', + '<<': '<<', + '>>': '>>', + '+': '+', + '-': '-', + '*': '*', + '/': '/', + '%': '%', + '||': '||' }; const forbiddenFunctions: Record = { diff --git a/packages/sync-rules/src/sync_plan/expression.ts b/packages/sync-rules/src/sync_plan/expression.ts new file mode 100644 index 000000000..0100af295 --- /dev/null +++ b/packages/sync-rules/src/sync_plan/expression.ts @@ -0,0 +1,219 @@ +/** + * An enumeration of all scalar SQL expressions supported by the sync service. + * + * Note that these expressions can be serialized, and the evaluation of serialized expressions must be stable. + * + * The `Data` type parameter encodes what external data references an {@link ExternalData} node may reference: Bucket + * data sources can only reference row data (e.g. to evaluate filters), while queriers can only reference connection + * data. Adding the parameter to expressions ensures the entire subtree can only reference expected sources. + */ +export type SqlExpression = + | ExternalData + | UnaryExpression + | BinaryExpression + | BetweenExpression + | ScalarInExpression + | CaseWhenExpression + | CastExpression + | ScalarFunctionCallExpression + | LiteralExpression; + +/** + * External data injected into the expression during evaluation. + * + * What kinds of external data is allowed depends on the expression. For instance, expressions for bucket data sources + * may only reference columns in the current row. + */ +export type ExternalData = { type: 'data'; source: Data }; + +export type UnaryOperator = 'not' | '~' | '+' | '-'; + +export type UnaryExpression = { + type: 'unary'; + operand: SqlExpression; + operator: UnaryOperator; +}; + +export type BinaryOperator = + | 'or' + | 'and' + | '=' + | 'is' + | '<' + | '<=' + | '>' + | '>=' + | '&' + | '|' + | '<<' + | '>>' + | '+' + | '-' + | '*' + | '/' + | '%' + | '||'; + +/** + * A binary expression in SQLite, `$left $op $right`. + * + * Note that the `LIKE`, `GLOB`, `REGEXP` and `MATCH`, `->` and `->>` operators are not represented as binary + * expressions but rather as the functions SQLite [would call for them](https://www.sqlite.org/lang_expr.html#the_like_glob_regexp_match_and_extract_operators). + * Also, note that negated operators (e.g. `!=` or `NOT IN`) are represented by wrapping the expression in a unary + * `NOT`. This makes transformations to DNF easier. + */ +export type BinaryExpression = { + type: 'binary'; + left: SqlExpression; + operator: BinaryOperator; + right: SqlExpression; +}; + +export type BetweenExpression = { + type: 'between'; + value: SqlExpression; + low: SqlExpression; + high: SqlExpression; +}; + +/** + * An `expr IN ($in0, $in1, ..., $inN)` expression. + */ +export type ScalarInExpression = { + type: 'scalar_in'; + target: SqlExpression; + in: SqlExpression[]; +}; + +/** + * A `CASE WHEN` expression as supported by SQLite. + */ +export type CaseWhenExpression = { + type: 'case_when'; + operand?: SqlExpression; + whens: { when: SqlExpression; then: SqlExpression }[]; + else?: SqlExpression; +}; + +/** + * A `CAST` expression. + */ +export type CastExpression = { + type: 'cast'; + operand: SqlExpression; + cast_as: 'string' | 'numeric' | 'real' | 'integer' | 'blob'; +}; + +/** + * A constant literal in SQL. + */ +export type LiteralExpression = + | { type: 'lit_null' } + | { type: 'lit_double'; value: number } + | { + type: 'lit_int'; + /** + * The integer value as a base 10 string. We don't use the correct `bigint` type to be able to serialize and + * deserialize with the default JSON implementation. + */ + base10: string; + } + | { type: 'lit_string'; value: string }; + +/** + * A scalar (non-aggregate, non-window, non-table-valued) function call in SQL. + */ +export type ScalarFunctionCallExpression = { + type: 'function'; + function: string; + parameters: SqlExpression[]; +}; + +export type ArgumentCount = number | { min: number; max?: number; mustBeEven?: boolean; mustBeOdd?: boolean }; + +export const supportedFunctions: Record = { + // https://sqlite.org/lang_corefunc.html#list_of_core_functions + abs: 1, + char: { min: 0 }, + coalesce: { min: 2 }, + concat: { min: 1 }, + concat_ws: { min: 2 }, + format: { min: 1 }, + glob: 2, + hex: 1, + ifnull: { min: 2 }, + if: { min: 2 }, + iif: { min: 2 }, + instr: 2, + length: 1, + // TODO: Establish defaults for case sensitivity, changing escape characters, ICU support. + // We might just want to remove LIKE support since we don't seem to have it in sql_functions.ts + like: { min: 2, max: 3 }, + // likelihood: 2, + // likely: 1, + lower: 1, + ltrim: { min: 1, max: 2 }, + max: { min: 2 }, + min: { min: 2 }, + nullif: 2, + octet_length: 1, + printf: { min: 1 }, + quote: 1, + replace: 3, + round: { min: 1, max: 2 }, + rtrim: { min: 1, max: 2 }, + sign: 1, + substr: { min: 2, max: 3 }, + substring: { min: 2, max: 3 }, + trim: { min: 1, max: 2 }, + typeof: 1, + unhex: { min: 1, max: 2 }, + unicode: 1, + unistr: 1, + unistr_quote: 1, + // unlikely: 1, + upper: 1, + zeroblob: 1, + // Scalar functions from https://sqlite.org/json1.html#overview + // We disallow jsonb functions because they're not implemented in sql_functions.ts and we want to preserve + // compatibility with that. + json: 1, + // jsonb: 1, + json_array: { min: 0 }, + // jsonb_array: { min: 0 }, + json_array_length: { min: 1, max: 2 }, + json_error_position: 1, + json_extract: { min: 2 }, + // jsonb_extract: { min: 2 }, + json_insert: { min: 3, mustBeOdd: true }, + // jsonb_insert: { min: 3, mustBeOdd: true }, + json_object: { min: 0, mustBeEven: true }, + // jsonb_object: { min: 0, mustBeEven: true }, + json_patch: 2, + // jsonb_patch: 2, + json_pretty: 1, + json_remove: { min: 2 }, + // jsonb_remove: { min: 2 }, + json_replace: { min: 3, mustBeOdd: true }, + // jsonb_replace: { min: 3, mustBeOdd: true }, + json_set: { min: 3, mustBeOdd: true }, + // jsonb_set: { min: 3, mustBeOdd: true }, + json_type: { min: 1, max: 2 }, + json_valid: { min: 1, max: 2 }, + json_quote: { min: 1 }, + + // https://www.sqlite.org/lang_datefunc.html, but we only support datetime and unixepoch + // TODO: Ensure our support matches the current unixepoch behavior in sql_functions.ts, register as user-defined + // function to patch default if in doubt. + unixepoch: { min: 1 }, + datetime: { min: 1 }, + + // PowerSync-specific, mentioned in https://docs.powersync.com/sync/rules/supported-sql#functions + base64: 1, + json_keys: 1, + uuid_blob: 1, + ST_AsGeoJSON: 1, + ST_AsText: 1, + ST_X: 1, + ST_: 1 +}; diff --git a/packages/sync-rules/src/sync_plan/expression_to_sql.ts b/packages/sync-rules/src/sync_plan/expression_to_sql.ts new file mode 100644 index 000000000..1e8d0c55a --- /dev/null +++ b/packages/sync-rules/src/sync_plan/expression_to_sql.ts @@ -0,0 +1,220 @@ +import { + BetweenExpression, + BinaryExpression, + BinaryOperator, + CaseWhenExpression, + CastExpression, + LiteralExpression, + ScalarFunctionCallExpression, + ScalarInExpression, + SqlExpression, + UnaryExpression, + UnaryOperator +} from './expression.js'; +import { ExpressionVisitor, visitExpr } from './expression_visitor.js'; + +/** + * Renders {@link SqlExpression}s into SQL text supported by SQLite. + */ +export class ExpressionToSqlite implements ExpressionVisitor { + sql = ''; + + private needsSpace = false; + + addExpression(expr: SqlExpression, precedence: Precedence | 0 = 0) { + visitExpr(this, expr, precedence); + } + + private space() { + this.sql += ' '; + } + + private addLexeme(text: string, options?: { spaceLeft?: boolean; spaceRight?: boolean }): number { + const spaceLeft = options?.spaceLeft ?? true; + const spaceRight = options?.spaceRight ?? true; + + if (this.needsSpace && spaceLeft) { + this.space(); + } + + const startOffset = this.sql.length; + this.sql += text; + this.needsSpace = spaceRight; + return startOffset; + } + + private identifier(name: string) { + this.addLexeme(`"${name.replaceAll('"', '""')}"`); + } + + private string(name: string) { + this.addLexeme(`'${name.replaceAll("'", "''")}'`); + } + + private commaSeparated(exprList: Iterable>) { + let first = true; + for (const expr of exprList) { + if (!first) { + this.addLexeme(',', { spaceLeft: false }); + } + + this.addExpression(expr); + first = false; + } + } + + private parenthesis(inner: () => void) { + this.addLexeme('(', { spaceRight: false }); + inner(); + this.addLexeme(')', { spaceLeft: false }); + } + + private maybeParenthesis(outerPrecedence: Precedence | 0, innerPrecedence: Precedence, inner: () => void) { + if (outerPrecedence > innerPrecedence) { + this.parenthesis(inner); + } else { + inner(); + } + } + + visitExternalData(): void { + this.addLexeme('?'); + } + + visitUnaryExpression(expr: UnaryExpression, outerPrecedence: Precedence | 0): void { + const innerPrecedence = unaryPrecedence[expr.operator]; + this.maybeParenthesis(outerPrecedence, innerPrecedence, () => { + this.addLexeme(expr.operator); + this.addExpression(expr.operand, innerPrecedence); + }); + } + + visitBinaryExpression(expr: BinaryExpression, outerPrecedence: Precedence | 0): void { + const innerPrecedence = binaryPrecedence[expr.operator]; + this.maybeParenthesis(outerPrecedence, innerPrecedence, () => { + this.addExpression(expr.left, innerPrecedence); + this.addLexeme(expr.operator); + this.addExpression(expr.right, innerPrecedence); + }); + } + + visitBetweenExpression(expr: BetweenExpression, outerPrecedence: 0 | Precedence): void { + const innerPrecedence = Precedence.equals; + this.maybeParenthesis(outerPrecedence, innerPrecedence, () => { + this.addExpression(expr.value, Precedence.equals); + this.addLexeme('BETWEEN'); + this.addExpression(expr.low, Precedence.equals); + this.addLexeme('AND'); + this.addExpression(expr.high, Precedence.equals); + }); + } + + visitScalarInExpression(expr: ScalarInExpression, arg: Precedence | 0): void { + if (expr.in.length == 0) { + // x IN () is invalid, but it can't be true either way. + this.addLexeme('FALSE'); + return; + } + + this.maybeParenthesis(arg, Precedence.equals, () => { + this.addExpression(expr.target, Precedence.equals); + this.addLexeme('IN'); + this.parenthesis(() => this.commaSeparated(expr.in)); + }); + } + + visitCaseWhenExpression(expr: CaseWhenExpression, arg: Precedence | 0): void { + this.addLexeme('CASE'); + if (expr.operand) { + this.addExpression(expr.operand); + } + for (const when of expr.whens) { + this.addLexeme('WHEN'); + this.addExpression(when.when); + this.addLexeme('THEN'); + this.addExpression(when.then); + } + + if (expr.else) { + this.addLexeme('ELSE'); + this.addExpression(expr.else); + } + this.addLexeme('END'); + } + + visitCastExpression(expr: CastExpression, arg: Precedence | 0): void { + this.addLexeme('CAST(', { spaceRight: false }); + this.addExpression(expr.operand); + this.addLexeme('AS'); + this.addLexeme(expr.cast_as); + this.addLexeme(')', { spaceLeft: false }); + } + + visitScalarFunctionCallExpression(expr: ScalarFunctionCallExpression, arg: Precedence | 0): void { + this.identifier(expr.function); + this.addLexeme('(', { spaceLeft: false, spaceRight: false }); + this.commaSeparated(expr.parameters); + this.addLexeme(')', { spaceLeft: false }); + } + + visitLiteralExpression(expr: LiteralExpression, arg: Precedence | 0): void { + if (expr.type == 'lit_null') { + this.addLexeme('NULL'); + } else if (expr.type == 'lit_double') { + this.addLexeme(expr.value.toString()); + } else if (expr.type == 'lit_int') { + this.addLexeme(expr.base10); + } else { + this.string(expr.value); + } + } + + static toSqlite(expr: SqlExpression): string { + const visitor = new ExpressionToSqlite(); + visitor.addExpression(expr); + return visitor.sql; + } +} + +enum Precedence { + or = 1, + and = 2, + not = 3, + equals = 4, + comparison = 5, + binary = 6, + addition = 7, + multiplication = 8, + concat = 9, + collate = 10, + unary = 11 +} + +// https://www.sqlite.org/lang_expr.html#operators_and_parse_affecting_attributes +const binaryPrecedence: Record = { + or: Precedence.or, + and: Precedence.and, + '=': Precedence.equals, + is: Precedence.equals, + '<': Precedence.comparison, + '>': Precedence.comparison, + '<=': Precedence.comparison, + '>=': Precedence.comparison, + '&': Precedence.binary, + '|': Precedence.binary, + '<<': Precedence.binary, + '>>': Precedence.binary, + '+': Precedence.addition, + '-': Precedence.addition, + '*': Precedence.multiplication, + '/': Precedence.multiplication, + '%': Precedence.multiplication, + '||': Precedence.concat +}; + +const unaryPrecedence: Record = { + not: Precedence.not, + '~': Precedence.unary, + '+': Precedence.unary, + '-': Precedence.unary +}; diff --git a/packages/sync-rules/src/sync_plan/expression_visitor.ts b/packages/sync-rules/src/sync_plan/expression_visitor.ts new file mode 100644 index 000000000..806bff450 --- /dev/null +++ b/packages/sync-rules/src/sync_plan/expression_visitor.ts @@ -0,0 +1,231 @@ +import { + BetweenExpression, + BinaryExpression, + CaseWhenExpression, + CastExpression, + ExternalData, + LiteralExpression, + ScalarFunctionCallExpression, + ScalarInExpression, + SqlExpression, + UnaryExpression +} from './expression.js'; + +/** + * Callbacks for each type of SQL expression we support. + * + * The {@link visit} function can be used to call the appropriate callback given an expression. + */ +export interface ExpressionVisitor { + visitExternalData(expr: ExternalData, arg: Arg): R; + visitUnaryExpression(expr: UnaryExpression, arg: Arg): R; + visitBinaryExpression(expr: BinaryExpression, arg: Arg): R; + visitBetweenExpression(expr: BetweenExpression, arg: Arg): R; + visitScalarInExpression(expr: ScalarInExpression, arg: Arg): R; + visitCaseWhenExpression(expr: CaseWhenExpression, arg: Arg): R; + visitCastExpression(expr: CastExpression, arg: Arg): R; + visitScalarFunctionCallExpression(expr: ScalarFunctionCallExpression, arg: Arg): R; + visitLiteralExpression(expr: LiteralExpression, arg: Arg): R; +} + +/** + * Invokes the appropriate visit method for the given expression. + */ +export function visitExpr( + visitor: ExpressionVisitor, + expr: SqlExpression, + arg: Arg +): R { + switch (expr.type) { + case 'data': + return visitor.visitExternalData(expr, arg); + case 'unary': + return visitor.visitUnaryExpression(expr, arg); + case 'between': + return visitor.visitBetweenExpression(expr, arg); + case 'binary': + return visitor.visitBinaryExpression(expr, arg); + case 'scalar_in': + return visitor.visitScalarInExpression(expr, arg); + case 'case_when': + return visitor.visitCaseWhenExpression(expr, arg); + case 'cast': + return visitor.visitCastExpression(expr, arg); + case 'function': + return visitor.visitScalarFunctionCallExpression(expr, arg); + default: + return visitor.visitLiteralExpression(expr, arg); + } +} + +/** + * A utility for traversing through a {@link SqlExpression} tree. + */ +export abstract class RecursiveExpressionVisitor implements ExpressionVisitor { + abstract defaultExpression(expr: SqlExpression, arg: Arg): R; + + /** + * Invokes the appropriate visit method for the given expression. + */ + visit(expr: SqlExpression, arg: Arg): R { + return visitExpr(this, expr, arg); + } + + visitChildren(expr: SqlExpression, arg: Arg) { + switch (expr.type) { + case 'data': + break; // No subexpression + case 'unary': + this.visit(expr.operand, arg); + break; + case 'between': + this.visit(expr.value, arg); + this.visit(expr.low, arg); + this.visit(expr.high, arg); + break; + case 'binary': + this.visit(expr.left, arg); + this.visit(expr.right, arg); + break; + case 'scalar_in': + this.visit(expr.target, arg); + for (const target of expr.in) { + this.visit(target, arg); + } + break; + case 'case_when': + return this.visitCaseWhenExpression(expr, arg); + case 'cast': + this.visit(expr.operand, arg); + break; + case 'function': + for (const param of expr.parameters) { + this.visit(param, arg); + } + break; + default: + // Literals have no subexpressions. + } + } + + visitExternalData(expr: ExternalData, arg: Arg) { + return this.defaultExpression(expr, arg); + } + + visitUnaryExpression(expr: UnaryExpression, arg: Arg) { + return this.defaultExpression(expr, arg); + } + + visitBinaryExpression(expr: BinaryExpression, arg: Arg) { + return this.defaultExpression(expr, arg); + } + + visitBetweenExpression(expr: BetweenExpression, arg: Arg) { + return this.defaultExpression(expr, arg); + } + + visitScalarInExpression(expr: ScalarInExpression, arg: Arg) { + return this.defaultExpression(expr, arg); + } + + visitCaseWhenExpression(expr: CaseWhenExpression, arg: Arg) { + return this.defaultExpression(expr, arg); + } + + visitCastExpression(expr: CastExpression, arg: Arg) { + return this.defaultExpression(expr, arg); + } + + visitScalarFunctionCallExpression(expr: ScalarFunctionCallExpression, arg: Arg) { + return this.defaultExpression(expr, arg); + } + + visitLiteralExpression(expr: LiteralExpression, arg: Arg) { + return this.defaultExpression(expr, arg); + } +} + +/** + * A visitor applying a mapping function to external data references in expressions. + */ +export class MapSourceVisitor implements ExpressionVisitor> { + constructor(private readonly map: (a: DataIn) => DataOut) {} + + visitExternalData(expr: ExternalData): SqlExpression { + return { type: 'data', source: this.map(expr.source) }; + } + + visitUnaryExpression(expr: UnaryExpression, arg: undefined): SqlExpression { + const operand = visitExpr(this, expr.operand, arg); + if (operand == expr.operand) { + return expr as SqlExpression; // unchanged subtree + } + + return { type: 'unary', operator: expr.operator, operand }; + } + + visitBinaryExpression(expr: BinaryExpression, arg: undefined): SqlExpression { + const left = visitExpr(this, expr.left, arg); + const right = visitExpr(this, expr.right, arg); + if (left == expr.left && right == expr.right) { + return expr as SqlExpression; // unchanged subtree + } + + return { type: 'binary', operator: expr.operator, left, right }; + } + + visitBetweenExpression(expr: BetweenExpression, arg: undefined): SqlExpression { + const value = visitExpr(this, expr.value, arg); + const low = visitExpr(this, expr.low, arg); + const high = visitExpr(this, expr.high, arg); + if (value == expr.value && expr.low == low && expr.high == high) { + return expr as SqlExpression; // unchanged subtree + } + + return { type: 'between', value, low, high }; + } + + visitScalarInExpression(expr: ScalarInExpression, arg: undefined): SqlExpression { + return { + type: 'scalar_in', + target: visitExpr(this, expr.target, arg), + in: expr.in.map((e) => visitExpr(this, e, arg)) + }; + } + + visitCaseWhenExpression(expr: CaseWhenExpression, arg: undefined): SqlExpression { + return { + type: 'case_when', + operand: expr.operand && visitExpr(this, expr.operand, arg), + whens: expr.whens.map(({ when, then }) => ({ + when: visitExpr(this, when, arg), + then: visitExpr(this, then, arg) + })), + else: expr.else && visitExpr(this, expr.else, arg) + }; + } + + visitCastExpression(expr: CastExpression, arg: undefined): SqlExpression { + const operand = visitExpr(this, expr.operand, arg); + if (operand == expr.operand) { + return expr as SqlExpression; // unchanged subtree + } + + return { type: 'cast', operand, cast_as: expr.cast_as }; + } + + visitScalarFunctionCallExpression( + expr: ScalarFunctionCallExpression, + arg: undefined + ): SqlExpression { + return { + type: 'function', + function: expr.function, + parameters: expr.parameters.map((p) => visitExpr(this, p, arg)) + }; + } + + visitLiteralExpression(expr: LiteralExpression, arg: undefined): SqlExpression { + return expr; + } +} diff --git a/packages/sync-rules/src/sync_plan/plan.ts b/packages/sync-rules/src/sync_plan/plan.ts index 933ca5e23..cb75e4eed 100644 --- a/packages/sync-rules/src/sync_plan/plan.ts +++ b/packages/sync-rules/src/sync_plan/plan.ts @@ -2,6 +2,7 @@ import { BucketPriority } from '../BucketDescription.js'; import { ParameterLookupScope } from '../HydrationState.js'; import { TablePattern } from '../TablePattern.js'; import { UnscopedEvaluatedParameters } from '../types.js'; +import { SqlExpression } from './expression.js'; /** * A compiled "sync plan", a description for @@ -162,25 +163,6 @@ export interface StreamQuerier { sourceInstantiation: ParameterValue[]; } -/** - * An expression that can be evaluated by SQLite. - * - * The type parameter `T` describes which values this expression uses. For instance, an expression in a - * {@link TableProcessor} would only use {@link ColumnSqlParameterValue}s since no request is available in that context. - */ -export interface SqlExpression { - /** - * To SQL expression to evaluate. - * - * The expression is guaranteed to not contain any column references. All dependencies to row data are encoded as SQL - * parameters that have an instantiation in {@link instantiation}. - * - * For instance, the stream `SELECT UPPER(name) FROM users;` would have `UPPER(?1)` as SQL and a - */ - sql: string; - values: T[]; -} - export type SqlParameterValue = ColumnSqlParameterValue | RequestSqlParameterValue; /** diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index d472e343f..82caabd0a 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -1,4 +1,5 @@ import { TablePattern } from '../TablePattern.js'; +import { SqlExpression } from './expression.js'; import { ColumnSource, ColumnSqlParameterValue, @@ -6,7 +7,6 @@ import { ParameterValue, PartitionKey, RequestSqlParameterValue, - SqlExpression, StreamBucketDataSource, StreamDataSource, StreamOptions, diff --git a/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap b/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap index 019c749dd..b35e31148 100644 --- a/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap +++ b/packages/sync-rules/test/src/compiler/__snapshots__/advanced.test.ts.snap @@ -18,12 +18,23 @@ exports[`new sync stream features > in array 1`] = ` ], "filters": [ { - "sql": "? IN ('public', 'archived')", - "values": [ + "in": [ { - "column": "state", + "type": "lit_string", + "value": "public", + }, + { + "type": "lit_string", + "value": "archived", }, ], + "target": { + "source": { + "column": "state", + }, + "type": "data", + }, + "type": "scalar_in", }, ], "hash": 240499768, @@ -80,12 +91,10 @@ exports[`new sync stream features > joins feedback > response 1 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -102,23 +111,19 @@ exports[`new sync stream features > joins feedback > response 1 1`] = ` "hash": 136074946, "output": [ { - "sql": "?", - "values": [ - { - "column": "group_id", - }, - ], + "source": { + "column": "group_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, }, ], @@ -133,23 +138,19 @@ exports[`new sync stream features > joins feedback > response 1 1`] = ` "hash": 322713396, "output": [ { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "group_id", - }, - ], + "source": { + "column": "group_id", + }, + "type": "data", }, }, ], @@ -171,12 +172,20 @@ exports[`new sync stream features > joins feedback > response 1 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -248,12 +257,10 @@ exports[`new sync stream features > joins feedback > response 4 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "organization_id", - }, - ], + "source": { + "column": "organization_id", + }, + "type": "data", }, }, ], @@ -268,34 +275,36 @@ exports[`new sync stream features > joins feedback > response 4 1`] = ` { "filters": [ { - "sql": "? = 'ORGANIZATION_LEADER'", - "values": [ - { + "left": { + "source": { "column": "role_id", }, - ], + "type": "data", + }, + "operator": "=", + "right": { + "type": "lit_string", + "value": "ORGANIZATION_LEADER", + }, + "type": "binary", }, ], "hash": 378594320, "output": [ { - "sql": "?", - "values": [ - { - "column": "organization_id", - }, - ], + "source": { + "column": "organization_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "account_id", - }, - ], + "source": { + "column": "account_id", + }, + "type": "data", }, }, ], @@ -317,12 +326,20 @@ exports[`new sync stream features > joins feedback > response 4 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -379,12 +396,10 @@ exports[`new sync stream features > joins feedback > response 5 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "family_id", - }, - ], + "source": { + "column": "family_id", + }, + "type": "data", }, }, ], @@ -404,12 +419,10 @@ exports[`new sync stream features > joins feedback > response 5 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -426,23 +439,19 @@ exports[`new sync stream features > joins feedback > response 5 1`] = ` "hash": 249133436, "output": [ { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "auth_id", - }, - ], + "source": { + "column": "auth_id", + }, + "type": "data", }, }, ], @@ -457,23 +466,19 @@ exports[`new sync stream features > joins feedback > response 5 1`] = ` "hash": 230646996, "output": [ { - "sql": "?", - "values": [ - { - "column": "family_id", - }, - ], + "source": { + "column": "family_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -495,12 +500,20 @@ exports[`new sync stream features > joins feedback > response 5 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -572,12 +585,10 @@ exports[`new sync stream features > joins feedback > response 8 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -594,23 +605,19 @@ exports[`new sync stream features > joins feedback > response 8 1`] = ` "hash": 283859073, "output": [ { - "sql": "?", - "values": [ - { - "column": "event_id", - }, - ], + "source": { + "column": "event_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, }, ], @@ -632,12 +639,20 @@ exports[`new sync stream features > joins feedback > response 8 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -693,12 +708,10 @@ exports[`new sync stream features > joins feedback > response 9 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -715,23 +728,19 @@ exports[`new sync stream features > joins feedback > response 9 1`] = ` "hash": 88350941, "output": [ { - "sql": "?", - "values": [ - { - "column": "organization_id", - }, - ], + "source": { + "column": "organization_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, }, ], @@ -746,23 +755,19 @@ exports[`new sync stream features > joins feedback > response 9 1`] = ` "hash": 343817501, "output": [ { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "organization_id", - }, - ], + "source": { + "column": "organization_id", + }, + "type": "data", }, }, ], @@ -784,12 +789,20 @@ exports[`new sync stream features > joins feedback > response 9 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -861,12 +874,10 @@ exports[`new sync stream features > joins feedback > response 10 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -883,23 +894,19 @@ exports[`new sync stream features > joins feedback > response 10 1`] = ` "hash": 88350941, "output": [ { - "sql": "?", - "values": [ - { - "column": "organization_id", - }, - ], + "source": { + "column": "organization_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, }, ], @@ -914,23 +921,19 @@ exports[`new sync stream features > joins feedback > response 10 1`] = ` "hash": 343817501, "output": [ { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "organization_id", - }, - ], + "source": { + "column": "organization_id", + }, + "type": "data", }, }, ], @@ -945,23 +948,19 @@ exports[`new sync stream features > joins feedback > response 10 1`] = ` "hash": 392582824, "output": [ { - "sql": "?", - "values": [ - { - "column": "address_id", - }, - ], + "source": { + "column": "address_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -983,12 +982,20 @@ exports[`new sync stream features > joins feedback > response 10 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.' || 'app_metadata.user_id'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "app_metadata.user_id", }, ], + "type": "function", }, "type": "request", }, @@ -1076,12 +1083,10 @@ exports[`new sync stream features > joins feedback > response 11 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -1098,23 +1103,19 @@ exports[`new sync stream features > joins feedback > response 11 1`] = ` "hash": 369143364, "output": [ { - "sql": "?", - "values": [ - { - "column": "ticket_id", - }, - ], + "source": { + "column": "ticket_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, }, ], @@ -1136,12 +1137,20 @@ exports[`new sync stream features > joins feedback > response 11 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -1197,12 +1206,10 @@ exports[`new sync stream features > joins feedback > response 13 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -1219,23 +1226,19 @@ exports[`new sync stream features > joins feedback > response 13 1`] = ` "hash": 496369348, "output": [ { - "sql": "?", - "values": [ - { - "column": "assignment_id", - }, - ], + "source": { + "column": "assignment_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "user_id", - }, - ], + "source": { + "column": "user_id", + }, + "type": "data", }, }, ], @@ -1248,34 +1251,36 @@ exports[`new sync stream features > joins feedback > response 13 1`] = ` { "filters": [ { - "sql": "? = TRUE", - "values": [ - { + "left": { + "source": { "column": "active", }, - ], + "type": "data", + }, + "operator": "=", + "right": { + "base10": "1", + "type": "lit_int", + }, + "type": "binary", }, ], - "hash": 189091889, + "hash": 221457926, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -1290,23 +1295,19 @@ exports[`new sync stream features > joins feedback > response 13 1`] = ` "hash": 532009492, "output": [ { - "sql": "?", - "values": [ - { - "column": "checkpoint_id", - }, - ], + "source": { + "column": "checkpoint_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "assignment_id", - }, - ], + "source": { + "column": "assignment_id", + }, + "type": "data", }, }, ], @@ -1328,12 +1329,20 @@ exports[`new sync stream features > joins feedback > response 13 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -1422,22 +1431,18 @@ exports[`new sync stream features > order-independent parameters 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "region", - }, - ], + "source": { + "column": "region", + }, + "type": "data", }, }, { "expr": { - "sql": "?", - "values": [ - { - "column": "org", - }, - ], + "source": { + "column": "org", + }, + "type": "data", }, }, ], @@ -1457,22 +1462,18 @@ exports[`new sync stream features > order-independent parameters 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "region", - }, - ], + "source": { + "column": "region", + }, + "type": "data", }, }, { "expr": { - "sql": "?", - "values": [ - { - "column": "org", - }, - ], + "source": { + "column": "org", + }, + "type": "data", }, }, ], @@ -1494,23 +1495,39 @@ exports[`new sync stream features > order-independent parameters 1`] = ` "sourceInstantiation": [ { "expr": { - "sql": "? ->> '$.' || 'region'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "subscription", + }, + "type": "data", + }, { - "request": "subscription", + "type": "lit_string", + "value": "region", }, ], + "type": "function", }, "type": "request", }, { "expr": { - "sql": "? ->> '$.' || 'org'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "org", }, ], + "type": "function", }, "type": "request", }, diff --git a/packages/sync-rules/test/src/compiler/__snapshots__/compatibility.test.ts.snap b/packages/sync-rules/test/src/compiler/__snapshots__/compatibility.test.ts.snap index e4ed234db..9ec824f62 100644 --- a/packages/sync-rules/test/src/compiler/__snapshots__/compatibility.test.ts.snap +++ b/packages/sync-rules/test/src/compiler/__snapshots__/compatibility.test.ts.snap @@ -22,12 +22,10 @@ exports[`old streams test > OR in subquery 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -44,23 +42,19 @@ exports[`old streams test > OR in subquery 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -73,23 +67,27 @@ exports[`old streams test > OR in subquery 1`] = ` { "filters": [ { - "sql": "? = 'test'", - "values": [ - { + "left": { + "source": { "column": "name", }, - ], + "type": "data", + }, + "operator": "=", + "right": { + "type": "lit_string", + "value": "test", + }, + "type": "binary", }, ], "hash": 139759831, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [], @@ -111,12 +109,20 @@ exports[`old streams test > OR in subquery 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -195,12 +201,10 @@ exports[`old streams test > in > on parameter data 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -225,12 +229,10 @@ exports[`old streams test > in > on parameter data 1`] = ` "functionName": "json_each", "outputs": [ { - "sql": "?", - "values": [ - { - "column": "value", - }, - ], + "source": { + "column": "value", + }, + "type": "data", }, ], "type": "table_valued", @@ -283,22 +285,18 @@ exports[`old streams test > in > on parameter data and table 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, { "expr": { - "sql": "?", - "values": [ - { - "column": "label", - }, - ], + "source": { + "column": "label", + }, + "type": "data", }, }, ], @@ -315,23 +313,19 @@ exports[`old streams test > in > on parameter data and table 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -353,12 +347,20 @@ exports[`old streams test > in > on parameter data and table 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -372,12 +374,10 @@ exports[`old streams test > in > on parameter data and table 1`] = ` "functionName": "json_each", "outputs": [ { - "sql": "?", - "values": [ - { - "column": "value", - }, - ], + "source": { + "column": "value", + }, + "type": "data", }, ], "type": "table_valued", @@ -438,12 +438,10 @@ exports[`old streams test > in > parameter and auth match on same column 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -468,12 +466,10 @@ exports[`old streams test > in > parameter and auth match on same column 1`] = ` "functionName": "json_each", "outputs": [ { - "sql": "?", - "values": [ - { - "column": "value", - }, - ], + "source": { + "column": "value", + }, + "type": "data", }, ], "type": "table_valued", @@ -487,12 +483,20 @@ exports[`old streams test > in > parameter and auth match on same column 1`] = ` "values": [ { "expr": { - "sql": "? ->> '$.' || 'issue'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "subscription", + }, + "type": "data", + }, { - "request": "subscription", + "type": "lit_string", + "value": "issue", }, ], + "type": "function", }, "type": "request", }, @@ -551,12 +555,10 @@ exports[`old streams test > in > parameter value in subquery 1`] = ` { "filters": [ { - "sql": "?", - "values": [ - { - "column": "is_admin", - }, - ], + "source": { + "column": "is_admin", + }, + "type": "data", }, ], "hash": 429775008, @@ -564,12 +566,10 @@ exports[`old streams test > in > parameter value in subquery 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -591,12 +591,20 @@ exports[`old streams test > in > parameter value in subquery 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -643,12 +651,10 @@ exports[`old streams test > in > row value in subquery 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -665,23 +671,19 @@ exports[`old streams test > in > row value in subquery 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -703,12 +705,20 @@ exports[`old streams test > in > row value in subquery 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -764,12 +774,10 @@ exports[`old streams test > in > two subqueries 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -786,23 +794,19 @@ exports[`old streams test > in > two subqueries 1`] = ` "hash": 75103629, "output": [ { - "sql": "?", - "values": [ - { - "column": "user_a", - }, - ], + "source": { + "column": "user_a", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "user_b", - }, - ], + "source": { + "column": "user_b", + }, + "type": "data", }, }, ], @@ -817,23 +821,19 @@ exports[`old streams test > in > two subqueries 1`] = ` "hash": 287627362, "output": [ { - "sql": "?", - "values": [ - { - "column": "user_b", - }, - ], + "source": { + "column": "user_b", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "user_a", - }, - ], + "source": { + "column": "user_a", + }, + "type": "data", }, }, ], @@ -855,12 +855,20 @@ exports[`old streams test > in > two subqueries 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -890,12 +898,20 @@ exports[`old streams test > in > two subqueries 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -951,12 +967,10 @@ exports[`old streams test > nested subqueries 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -971,23 +985,19 @@ exports[`old streams test > nested subqueries 1`] = ` { "filters": [ { - "sql": "?", - "values": [ - { - "column": "is_admin", - }, - ], + "source": { + "column": "is_admin", + }, + "type": "data", }, ], "hash": 429775008, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [], @@ -1002,23 +1012,19 @@ exports[`old streams test > nested subqueries 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -1108,12 +1114,24 @@ exports[`old streams test > normalization > distribute and 1`] = ` ], "filters": [ { - "sql": ""length"(?) > 2", - "values": [ - { - "column": "content", - }, - ], + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "content", + }, + "type": "data", + }, + ], + "type": "function", + }, + "operator": ">", + "right": { + "base10": "2", + "type": "lit_int", + }, + "type": "binary", }, ], "hash": 359214091, @@ -1121,12 +1139,10 @@ exports[`old streams test > normalization > distribute and 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -1142,12 +1158,24 @@ exports[`old streams test > normalization > distribute and 1`] = ` ], "filters": [ { - "sql": ""length"(?) > 2", - "values": [ - { - "column": "content", - }, - ], + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "content", + }, + "type": "data", + }, + ], + "type": "function", + }, + "operator": ">", + "right": { + "base10": "2", + "type": "lit_int", + }, + "type": "binary", }, ], "hash": 16883790, @@ -1166,23 +1194,19 @@ exports[`old streams test > normalization > distribute and 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -1204,12 +1228,20 @@ exports[`old streams test > normalization > distribute and 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -1236,12 +1268,20 @@ exports[`old streams test > normalization > distribute and 1`] = ` "lookupStages": [], "requestFilters": [ { - "sql": "? ->> '$.' || 'is_admin'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "is_admin", }, ], + "type": "function", }, ], "sourceInstantiation": [], @@ -1280,12 +1320,10 @@ exports[`old streams test > normalization > double negation 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -1302,23 +1340,19 @@ exports[`old streams test > normalization > double negation 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -1340,12 +1374,20 @@ exports[`old streams test > normalization > double negation 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -1390,7 +1432,7 @@ exports[`old streams test > normalization > negated and 1`] = ` "uniqueName": "stream|0", }, { - "hash": 299253648, + "hash": 120391031, "sources": [ 1, ], @@ -1408,12 +1450,10 @@ exports[`old streams test > normalization > negated and 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -1429,15 +1469,31 @@ exports[`old streams test > normalization > negated and 1`] = ` ], "filters": [ { - "sql": "NOT "length"(?) = 5", - "values": [ - { - "column": "content", + "operand": { + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "content", + }, + "type": "data", + }, + ], + "type": "function", }, - ], + "operator": "=", + "right": { + "base10": "5", + "type": "lit_int", + }, + "type": "binary", + }, + "operator": "not", + "type": "unary", }, ], - "hash": 27211305, + "hash": 395918337, "outputTableName": "comments", "partitionBy": [], "table": { @@ -1453,23 +1509,19 @@ exports[`old streams test > normalization > negated and 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -1491,12 +1543,20 @@ exports[`old streams test > normalization > negated and 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -1540,7 +1600,7 @@ exports[`old streams test > normalization > negated or 1`] = ` { "buckets": [ { - "hash": 160708722, + "hash": 117838948, "sources": [ 0, ], @@ -1554,25 +1614,39 @@ exports[`old streams test > normalization > negated or 1`] = ` ], "filters": [ { - "sql": "NOT "length"(?) = 5", - "values": [ - { - "column": "content", + "operand": { + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "content", + }, + "type": "data", + }, + ], + "type": "function", }, - ], + "operator": "=", + "right": { + "base10": "5", + "type": "lit_int", + }, + "type": "binary", + }, + "operator": "not", + "type": "unary", }, ], - "hash": 6712104, + "hash": 261122324, "outputTableName": "comments", "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -1589,23 +1663,19 @@ exports[`old streams test > normalization > negated or 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -1627,12 +1697,20 @@ exports[`old streams test > normalization > negated or 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -1695,12 +1773,10 @@ exports[`old streams test > or > parameter match or request condition 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -1736,12 +1812,20 @@ exports[`old streams test > or > parameter match or request condition 1`] = ` "sourceInstantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -1752,12 +1836,20 @@ exports[`old streams test > or > parameter match or request condition 1`] = ` "lookupStages": [], "requestFilters": [ { - "sql": "? ->> '$.' || 'is_admin'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "is_admin", }, ], + "type": "function", }, ], "sourceInstantiation": [], @@ -1803,12 +1895,10 @@ exports[`old streams test > or > parameter match or row condition 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -1824,12 +1914,24 @@ exports[`old streams test > or > parameter match or row condition 1`] = ` ], "filters": [ { - "sql": ""length"(?) = 3", - "values": [ - { - "column": "name", - }, - ], + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "name", + }, + "type": "data", + }, + ], + "type": "function", + }, + "operator": "=", + "right": { + "base10": "3", + "type": "lit_int", + }, + "type": "binary", }, ], "hash": 280456815, @@ -1853,12 +1955,20 @@ exports[`old streams test > or > parameter match or row condition 1`] = ` "sourceInstantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ { - "request": "auth", + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -1918,15 +2028,40 @@ exports[`old streams test > or > request condition or request condition 1`] = ` "lookupStages": [], "requestFilters": [ { - "sql": "? ->> '$.' || 'a' OR ? ->> 'b'", - "values": [ - { - "request": "auth", - }, - { - "request": "auth", - }, - ], + "left": { + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "a", + }, + ], + "type": "function", + }, + "operator": "or", + "right": { + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, + { + "type": "lit_string", + "value": "b", + }, + ], + "type": "function", + }, + "type": "binary", }, ], "sourceInstantiation": [], @@ -1982,12 +2117,24 @@ exports[`old streams test > or > row condition or parameter condition 1`] = ` ], "filters": [ { - "sql": ""length"(?) > 5", - "values": [ - { - "column": "content", - }, - ], + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "content", + }, + "type": "data", + }, + ], + "type": "function", + }, + "operator": ">", + "right": { + "base10": "5", + "type": "lit_int", + }, + "type": "binary", }, ], "hash": 58661560, @@ -2009,12 +2156,20 @@ exports[`old streams test > or > row condition or parameter condition 1`] = ` "lookupStages": [], "requestFilters": [ { - "sql": "? ->> '$.' || 'is_admin'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "is_admin", }, ], + "type": "function", }, ], "sourceInstantiation": [], @@ -2041,7 +2196,7 @@ exports[`old streams test > or > row condition or row condition 1`] = ` { "buckets": [ { - "hash": 409997171, + "hash": 531974709, "sources": [ 0, ], @@ -2055,18 +2210,51 @@ exports[`old streams test > or > row condition or row condition 1`] = ` ], "filters": [ { - "sql": ""length"(?) > 5 OR "json_array_length"(?) > 1", - "values": [ - { - "column": "content", + "left": { + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "content", + }, + "type": "data", + }, + ], + "type": "function", }, - { - "column": "tagged_users", + "operator": ">", + "right": { + "base10": "5", + "type": "lit_int", }, - ], + "type": "binary", + }, + "operator": "or", + "right": { + "left": { + "function": "json_array_length", + "parameters": [ + { + "source": { + "column": "tagged_users", + }, + "type": "data", + }, + ], + "type": "function", + }, + "operator": ">", + "right": { + "base10": "1", + "type": "lit_int", + }, + "type": "binary", + }, + "type": "binary", }, ], - "hash": 24706075, + "hash": 422901842, "outputTableName": "comments", "partitionBy": [], "table": { @@ -2127,12 +2315,10 @@ exports[`old streams test > or > subquery or token parameter 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -2163,23 +2349,19 @@ exports[`old streams test > or > subquery or token parameter 1`] = ` "hash": 473033600, "output": [ { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "owner_id", - }, - ], + "source": { + "column": "owner_id", + }, + "type": "data", }, }, ], @@ -2201,12 +2383,20 @@ exports[`old streams test > or > subquery or token parameter 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, @@ -2233,12 +2423,20 @@ exports[`old streams test > or > subquery or token parameter 1`] = ` "lookupStages": [], "requestFilters": [ { - "sql": "? ->> '$.' || 'is_admin'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "is_admin", }, ], + "type": "function", }, ], "sourceInstantiation": [], @@ -2273,12 +2471,24 @@ exports[`old streams test > row condition 1`] = ` ], "filters": [ { - "sql": ""length"(?) > 5", - "values": [ - { - "column": "content", - }, - ], + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "content", + }, + "type": "data", + }, + ], + "type": "function", + }, + "operator": ">", + "right": { + "base10": "5", + "type": "lit_int", + }, + "type": "binary", }, ], "hash": 58661560, @@ -2331,12 +2541,24 @@ exports[`old streams test > row filter and stream parameter 1`] = ` ], "filters": [ { - "sql": ""length"(?) > 5", - "values": [ - { - "column": "content", - }, - ], + "left": { + "function": "length", + "parameters": [ + { + "source": { + "column": "content", + }, + "type": "data", + }, + ], + "type": "function", + }, + "operator": ">", + "right": { + "base10": "5", + "type": "lit_int", + }, + "type": "binary", }, ], "hash": 226522950, @@ -2344,12 +2566,10 @@ exports[`old streams test > row filter and stream parameter 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -2371,12 +2591,20 @@ exports[`old streams test > row filter and stream parameter 1`] = ` "sourceInstantiation": [ { "expr": { - "sql": "? ->> '$.' || 'id'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "subscription", + }, + "type": "data", + }, { - "request": "subscription", + "type": "lit_string", + "value": "id", }, ], + "type": "function", }, "type": "request", }, @@ -2416,12 +2644,10 @@ exports[`old streams test > stream parameter 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "issue_id", - }, - ], + "source": { + "column": "issue_id", + }, + "type": "data", }, }, ], @@ -2443,12 +2669,20 @@ exports[`old streams test > stream parameter 1`] = ` "sourceInstantiation": [ { "expr": { - "sql": "? ->> '$.' || 'id'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "subscription", + }, + "type": "data", + }, { - "request": "subscription", + "type": "lit_string", + "value": "id", }, ], + "type": "function", }, "type": "request", }, @@ -2488,12 +2722,10 @@ exports[`old streams test > table alias 1`] = ` "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "account_id", - }, - ], + "source": { + "column": "account_id", + }, + "type": "data", }, }, ], @@ -2510,23 +2742,19 @@ exports[`old streams test > table alias 1`] = ` "hash": 405169598, "output": [ { - "sql": "?", - "values": [ - { - "column": "account_id", - }, - ], + "source": { + "column": "account_id", + }, + "type": "data", }, ], "partitionBy": [ { "expr": { - "sql": "?", - "values": [ - { - "column": "id", - }, - ], + "source": { + "column": "id", + }, + "type": "data", }, }, ], @@ -2548,12 +2776,20 @@ exports[`old streams test > table alias 1`] = ` "instantiation": [ { "expr": { - "sql": "? ->> '$.sub'", - "values": [ + "function": "->>", + "parameters": [ + { + "source": { + "request": "auth", + }, + "type": "data", + }, { - "request": "auth", + "type": "lit_string", + "value": "$.sub", }, ], + "type": "function", }, "type": "request", }, diff --git a/packages/sync-rules/test/src/compiler/sqlite.test.ts b/packages/sync-rules/test/src/compiler/sqlite.test.ts index 45c5a0016..cc5dd2532 100644 --- a/packages/sync-rules/test/src/compiler/sqlite.test.ts +++ b/packages/sync-rules/test/src/compiler/sqlite.test.ts @@ -2,12 +2,14 @@ import { Expr, parse } from 'pgsql-ast-parser'; import { PostgresToSqlite } from '../../../src/compiler/sqlite.js'; import { describe, expect, test } from 'vitest'; import { getLocation } from '../../../src/errors.js'; +import { ExpressionToSqlite } from '../../../src/sync_plan/expression_to_sql.js'; +import { NodeLocations } from '../../../src/compiler/expression.js'; describe('sqlite conversion', () => { test('literals', () => { expectNoErrors('null', 'NULL'); - expectNoErrors('true', 'TRUE'); - expectNoErrors('false', 'FALSE'); + expectNoErrors('true', '1'); + expectNoErrors('false', '0'); expectNoErrors("''", "''"); expectNoErrors("'hello world'", "'hello world'"); @@ -26,8 +28,10 @@ describe('sqlite conversion', () => { }); test('in values', () => { - expectNoErrors('"intrinsic:contains"(1)', 'FALSE'); - expectNoErrors('"intrinsic:contains"(1, 2, 3, 4)', '1 IN (2, 3, 4)'); + expectNoErrors('1 IN ARRAY[1,2,3]', '1 IN (1, 2, 3)'); + expectNoErrors('1 IN ARRAY[]', 'FALSE'); + + expectNoErrors('1 NOT IN ROW(1, 2, 3)', 'not 1 IN (1, 2, 3)'); }); test('precedence', () => { @@ -51,13 +55,13 @@ describe('sqlite conversion', () => { }); test('extract', () => { - expectNoErrors('1 ->> 2', '1 ->> 2'); - expectNoErrors("1 -> '$.foo'", "1 -> '$.foo'"); + expectNoErrors('1 ->> 2', '"->>"(1, 2)'); + expectNoErrors("1 -> '$.foo'", `"->"(1, '$.foo')`); }); test('unary', () => { - expectNoErrors('1 IS NOT NULL', '1 IS NOT NULL'); - expectNoErrors('NOT 1', 'NOT 1'); + expectNoErrors('1 IS NOT NULL', 'not 1 is NULL'); + expectNoErrors('NOT 1', 'not 1'); }); describe('errors', () => { @@ -183,17 +187,29 @@ function translate(source: string): [string, TranslationError[]] { const expr = parse(source, { entry: 'expr', locationTracking: true })[0] as Expr; const errors: TranslationError[] = []; - const translator = new PostgresToSqlite( - source, - { + const translator = new PostgresToSqlite({ + originalText: source, + errors: { report: (message, location) => { const resolved = getLocation(location); errors.push({ message, source: source.substring(resolved?.start ?? 0, resolved?.end) }); } }, - [] - ); - translator.addExpression(expr); - return [translator.sql, errors]; + locations: new NodeLocations(), + resolveTableName() { + throw new Error('unsupported in tests'); + }, + generateTableAlias() { + throw new Error('unsupported in tests'); + }, + joinSubqueryExpression(expr) { + return null; + } + }); + + const expression = translator.translateExpression(expr); + const toSql = ExpressionToSqlite.toSqlite(expression.node); + + return [toSql, errors]; }