diff --git a/cdk-postgresql/lib/handler.ts b/cdk-postgresql/lib/handler.ts index 526d5ff..2d6a816 100644 --- a/cdk-postgresql/lib/handler.ts +++ b/cdk-postgresql/lib/handler.ts @@ -4,6 +4,7 @@ import { VError } from "verror"; import { handler as dbHandler } from "./database.handler"; import { handler as roleHandler } from "./role.handler"; import { handler as roleMembershipHandler } from "./role-membership.handler"; +import { handler as replicationSlotHandler } from "./replication-slot.handler"; export const handler = async (event: CloudFormationCustomResourceEvent) => { switch (event.ResourceType) { @@ -13,6 +14,8 @@ export const handler = async (event: CloudFormationCustomResourceEvent) => { return dbHandler(event); case "Custom::Postgresql-RoleMembership": return roleMembershipHandler(event); + case "Custom::Postgresql-ReplicationSlot": + return replicationSlotHandler(event); default: throw new VError(`unexpected ResourceType: ${event.ResourceType}`); } diff --git a/cdk-postgresql/lib/index.ts b/cdk-postgresql/lib/index.ts index c37551a..c2cf3b2 100644 --- a/cdk-postgresql/lib/index.ts +++ b/cdk-postgresql/lib/index.ts @@ -1,4 +1,5 @@ export * from "./database"; export * from "./role"; export * from "./role-membership"; +export * from "./replication-slot"; export * from "./provider"; diff --git a/cdk-postgresql/lib/postgres.ts b/cdk-postgresql/lib/postgres.ts index 67dfeed..1aa93f1 100644 --- a/cdk-postgresql/lib/postgres.ts +++ b/cdk-postgresql/lib/postgres.ts @@ -105,3 +105,34 @@ export const revokeRoleMembership = async (props: { console.warn(thrown.message); } }; + +export const createReplicationSlot = async (props: { + client: Client; + name: string; + plugin: string; +}) => { + const { client, name, plugin } = props; + + await client.query("SELECT pg_create_logical_replication_slot($1, $2)", [ + name, + plugin, + ]); +}; + +/** + * Dropping is scoped to the connected database. A slot name is unique across + * the whole cluster, so an unscoped drop would destroy a slot decoding another + * database, and a dropped slot loses its replication position permanently. + * A slot that is already gone is left alone. + */ +export const dropReplicationSlot = async (props: { + client: Client; + name: string; +}) => { + const { client, name } = props; + + await client.query( + "SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1 AND database = current_database()", + [name] + ); +}; diff --git a/cdk-postgresql/lib/replication-slot.handler.ts b/cdk-postgresql/lib/replication-slot.handler.ts new file mode 100644 index 0000000..2dd73e3 --- /dev/null +++ b/cdk-postgresql/lib/replication-slot.handler.ts @@ -0,0 +1,216 @@ +import { + CloudFormationCustomResourceEvent, + CloudFormationCustomResourceCreateEvent, + CloudFormationCustomResourceUpdateEvent, + CloudFormationCustomResourceDeleteEvent, +} from "aws-lambda/trigger/cloudformation-custom-resource"; + +import { validateConnection, hashCode, getConnectedClient } from "./util"; +import { Connection } from "./lambda.types"; +import * as postgres from "./postgres"; + +interface Props { + ServiceToken: string; + Connection: Connection; + Name: string; + Plugin: string; + RequiredPublication?: string; +} + +const SLOT_PROVENANCES = ["created", "adopted"] as const; + +/** + * Whether this resource created the slot or adopted one that already existed. + * It is recorded in the physical id, which CloudFormation hands back verbatim + * on delete, so only a slot this resource created is ever dropped. + */ +type SlotProvenance = (typeof SLOT_PROVENANCES)[number]; + +export const handler = async (event: CloudFormationCustomResourceEvent) => { + switch (event.RequestType) { + case "Create": + return handleCreate(event); + case "Update": + return handleUpdate(event); + case "Delete": + return handleDelete(event); + } +}; + +const handleCreate = async (event: CloudFormationCustomResourceCreateEvent) => { + const props = event.ResourceProperties as Props; + validateProps(props); + const provenance = await createSlotIfAbsent(props); + return { + PhysicalResourceId: buildPhysicalId({ + identity: deriveSlotIdentity(props), + provenance, + }), + }; +}; + +/** + * A slot cannot be altered in place: the database it decodes and its name are + * its identity, and its plugin is fixed at creation. A change to either half of + * that identity reports a new physical id, so CloudFormation creates the new + * slot and then deletes the old resource. A plugin change under an unchanged + * identity is refused, because the existing slot would silently keep its old + * plugin. An unchanged identity keeps the physical id it was given, so a slot + * adopted at creation is never recorded as created. + */ +const handleUpdate = async (event: CloudFormationCustomResourceUpdateEvent) => { + const props = event.ResourceProperties as Props; + validateProps(props); + + const oldProps = event.OldResourceProperties as Props; + const identity = deriveSlotIdentity(props); + const { identity: currentIdentity } = parsePhysicalId( + event.PhysicalResourceId + ); + + if (identity != currentIdentity) { + const provenance = await createSlotIfAbsent(props); + return { PhysicalResourceId: buildPhysicalId({ identity, provenance }) }; + } + + if (props.Plugin != oldProps.Plugin) { + throw new Error( + `The plugin of replication slot "${props.Name}" cannot change in place; drop and recreate the slot deliberately instead` + ); + } + + return { PhysicalResourceId: event.PhysicalResourceId }; +}; + +/** + * Only a slot the physical id records this resource as having created is + * dropped. A slot that was adopted, or one whose physical id carries no + * provenance, may belong to someone else, and a dropped slot loses its + * replication position permanently. + */ +const handleDelete = async (event: CloudFormationCustomResourceDeleteEvent) => { + const props = event.ResourceProperties as Props; + validateProps(props); + + const { provenance } = parsePhysicalId(event.PhysicalResourceId); + if (provenance != "created") { + console.log( + "Not dropping replication slot, this resource did not create it", + props.Name, + provenance + ); + return {}; + } + + console.log("Dropping replication slot", props.Name); + const client = await getConnectedClient(props.Connection); + + try { + await postgres.dropReplicationSlot({ client, name: props.Name }); + } finally { + await client.end(); + } + + return {}; +}; + +const validateProps = (props: Props) => { + if (!("Connection" in props)) { + throw "Connection property is required"; + } + validateConnection(props.Connection); + + if (!("Name" in props)) { + throw "Name property is required"; + } + if (!("Plugin" in props)) { + throw "Plugin property is required"; + } +}; + +const deriveSlotIdentity = (props: Props): string => { + const { Host, Port, Database } = props.Connection; + const suffix = Math.abs( + hashCode(`${Host}-${Port}-${Database}-${props.Name}`) + ); + return `replication-slot-${suffix}`; +}; + +const buildPhysicalId = (params: { + identity: string; + provenance: SlotProvenance; +}): string => `${params.identity}-${params.provenance}`; + +/** + * The one place that knows the shape of a physical id. An id carrying no + * provenance suffix comes from the rollback of a failed create, and the guards + * in the create path throw exactly when a slot this resource never created + * already exists, so such an id reports "unknown" rather than "created". + */ +const parsePhysicalId = ( + physicalResourceId: string +): { identity: string; provenance: SlotProvenance | "unknown" } => { + for (const provenance of SLOT_PROVENANCES) { + const suffix = `-${provenance}`; + if (physicalResourceId.endsWith(suffix)) { + return { + identity: physicalResourceId.slice(0, -suffix.length), + provenance, + }; + } + } + + return { identity: physicalResourceId, provenance: "unknown" }; +}; + +const createSlotIfAbsent = async (props: Props): Promise => { + const client = await getConnectedClient(props.Connection); + + try { + if (props.RequiredPublication) { + const { rows } = await client.query( + "SELECT 1 FROM pg_publication WHERE pubname = $1", + [props.RequiredPublication] + ); + if (rows.length === 0) { + // A slot created before its publication decodes from a catalog snapshot + // in which the publication does not exist, and replication then fails + // continuously, so refusing here turns a subtle runtime failure into a + // loud deploy failure: + throw new Error( + `publication "${props.RequiredPublication}" does not exist in database "${props.Connection.Database}"; deploy the migration that creates it before this slot` + ); + } + } + + // `pg_replication_slots` lists the whole cluster, so a slot of the same + // name decoding another database must not be adopted as this one; letting + // Postgres raise "replication slot already exists" is the loud failure: + const { rows: existingSlots } = await client.query( + "SELECT plugin FROM pg_replication_slots WHERE slot_name = $1 AND database = current_database()", + [props.Name] + ); + const [existingSlot] = existingSlots; + + if (!existingSlot) { + console.log("Creating replication slot", props.Name); + await postgres.createReplicationSlot({ + client, + name: props.Name, + plugin: props.Plugin, + }); + return "created"; + } + + if (existingSlot.plugin !== props.Plugin) { + throw new Error( + `replication slot "${props.Name}" already exists in database "${props.Connection.Database}" with plugin "${existingSlot.plugin}" instead of "${props.Plugin}"; a slot's plugin is fixed at creation, so drop and recreate the slot deliberately instead` + ); + } + + console.log("Replication slot already exists", props.Name); + return "adopted"; + } finally { + await client.end(); + } +}; diff --git a/cdk-postgresql/lib/replication-slot.ts b/cdk-postgresql/lib/replication-slot.ts new file mode 100644 index 0000000..7231a6e --- /dev/null +++ b/cdk-postgresql/lib/replication-slot.ts @@ -0,0 +1,75 @@ +import { Construct } from "constructs"; +import * as cdk from "aws-cdk-lib"; +import { RemovalPolicy } from "aws-cdk-lib"; +import { Provider } from "./provider"; + +export interface ReplicationSlotProps { + /** + * Provider required to connect to the Postgresql server. Logical slots decode + * one database, the one the provider's `database` prop names. + */ + provider: Provider; + + /** + * The name of the slot. Must be unique on the PostgreSQL server instance + * where it is configured. + */ + name: string; + + /** + * The logical decoding plugin the slot is created with. It cannot change + * after creation. + * + * @default - "pgoutput" + */ + plugin?: string; + + /** + * A publication that has to exist before the slot is created. A slot created + * earlier decodes from a catalog snapshot in which the publication does not + * exist, and replication fails continuously, so creation is refused until + * the publication is there. + */ + requiredPublication?: string; + + /** + * Policy to apply when the slot is removed from this stack. A dropped slot + * loses its position permanently, so stacks that replicate production data + * should retain it. + * + * @default - The slot will be dropped. + */ + removalPolicy?: RemovalPolicy; +} + +/** + * A PostgreSQL logical replication slot. The database retains write-ahead-log + * data until the slot's consumer confirms having read it, which is what makes + * replication resumable after the consumer restarts. + * + * A slot that already exists on the same database with the same plugin is + * adopted and managed in place, and removing the resource only ever drops a + * slot this resource itself created. + */ +export class ReplicationSlot extends Construct { + constructor(scope: Construct, id: string, props: ReplicationSlotProps) { + super(scope, id); + + const { provider, name, plugin, requiredPublication, removalPolicy } = props; + + const cr = new cdk.CustomResource(this, "CustomResource", { + serviceToken: provider.serviceToken, + resourceType: "Custom::Postgresql-ReplicationSlot", + properties: { + connection: provider.buildConnectionProperty(), + name, + plugin: plugin ?? "pgoutput", + requiredPublication, + }, + pascalCaseProperties: true, + }); + + cr.applyRemovalPolicy(removalPolicy || cdk.RemovalPolicy.DESTROY); + cr.node.addDependency(provider); + } +} diff --git a/cdk-postgresql/package.json b/cdk-postgresql/package.json index 1c801bf..2572706 100644 --- a/cdk-postgresql/package.json +++ b/cdk-postgresql/package.json @@ -1,6 +1,6 @@ { "name": "@botpress/cdk-postgresql", - "version": "3.0.0", + "version": "3.1.0", "description": "Postgresql constructs for AWS CDK", "main": "./dist/index.cjs", "types": "./dist/index.d.cts", diff --git a/cdk-postgresql/test/helpers.ts b/cdk-postgresql/test/helpers.ts index b5df144..5e89ffd 100644 --- a/cdk-postgresql/test/helpers.ts +++ b/cdk-postgresql/test/helpers.ts @@ -81,3 +81,14 @@ export const isMemberOf = async (props: { ); return rows[0].is_member; }; + +export const replicationSlotExists = async ( + client: Client, + name: string +): Promise => { + const { rows } = await client.query( + "SELECT 1 FROM pg_replication_slots WHERE slot_name = $1 AND database = current_database()", + [name] + ); + return rows.length > 0; +}; diff --git a/cdk-postgresql/test/lambda.integration.test.ts b/cdk-postgresql/test/lambda.integration.test.ts index 884146b..426fc6a 100644 --- a/cdk-postgresql/test/lambda.integration.test.ts +++ b/cdk-postgresql/test/lambda.integration.test.ts @@ -13,7 +13,16 @@ import { import { Client } from "pg"; import { createDatabase, createRole } from "../lib/postgres"; import { handler as roleMembershipHandler } from "../lib/role-membership.handler"; -import { createSecret, dbExists, getDbOwner, isMemberOf, roleExists } from "./helpers"; +import { handler as replicationSlotHandler } from "../lib/replication-slot.handler"; +import { CloudFormationCustomResourceEvent } from "aws-lambda/trigger/cloudformation-custom-resource"; +import { + createSecret, + dbExists, + getDbOwner, + isMemberOf, + replicationSlotExists, + roleExists, +} from "./helpers"; import { secretsmanager } from "../lib/util"; import { beforeEach, afterEach, describe, test, expect, vi } from "vitest"; import { createRequire } from "node:module"; @@ -39,6 +48,8 @@ beforeEach(async () => { pgContainer = await new GenericContainer("postgres:16") .withExposedPorts(DB_PORT) .withEnvironment({ POSTGRES_PASSWORD: DB_MASTER_PASSWORD }) + // Logical replication slots require logical write-ahead-log decoding: + .withCommand(["postgres", "-c", "wal_level=logical"]) .start(); localstackContainer = await new GenericContainer("localstack/localstack:3") .withEnvironment({ SERVICES: "secretsmanager" }) @@ -681,3 +692,306 @@ describe("built lambda asset", () => { expect(rows[0].current_user).toEqual(roleName); }); }); + +describe("replication-slot handler", () => { + const SLOT_NAME = "test_slot"; + const OTHER_DB = "other_db"; + const CREATED_PHYSICAL_ID = "replication-slot-1-created"; + const PHYSICAL_ID_WITHOUT_PROVENANCE = "replication-slot-1"; + + const buildConnection = (database = DB_DEFAULT_DB) => ({ + Host: pgHost, + Port: pgPort, + Username: DB_MASTER_USERNAME, + PasswordArn: masterPasswordArn, + Database: database, + SSLMode: "disable" as const, + }); + + const buildCreateEvent = (props?: { requiredPublication?: string }) => + ({ + RequestType: "Create", + ResourceProperties: { + ServiceToken: "token", + Connection: buildConnection(), + Name: SLOT_NAME, + Plugin: "pgoutput", + RequiredPublication: props?.requiredPublication, + }, + }) as unknown as CloudFormationCustomResourceEvent; + + const buildUpdateEvent = (props: { + physicalResourceId: string; + name?: string; + plugin?: string; + database?: string; + }) => + ({ + RequestType: "Update", + PhysicalResourceId: props.physicalResourceId, + ResourceProperties: { + ServiceToken: "token", + Connection: buildConnection(props.database), + Name: props.name ?? SLOT_NAME, + Plugin: props.plugin ?? "pgoutput", + }, + OldResourceProperties: { + ServiceToken: "token", + Connection: buildConnection(), + Name: SLOT_NAME, + Plugin: "pgoutput", + }, + }) as unknown as CloudFormationCustomResourceEvent; + + const buildDeleteEvent = (physicalResourceId: string) => + ({ + RequestType: "Delete", + PhysicalResourceId: physicalResourceId, + ResourceProperties: { + ServiceToken: "token", + Connection: buildConnection(), + Name: SLOT_NAME, + Plugin: "pgoutput", + }, + }) as unknown as CloudFormationCustomResourceEvent; + + const createSlotAndReturnPhysicalId = async () => { + const response = await replicationSlotHandler(buildCreateEvent()); + return (response as { PhysicalResourceId: string }).PhysicalResourceId; + }; + + const connectAs = async (database: string) => { + const client = new Client({ + host: pgHost, + port: pgPort, + user: DB_MASTER_USERNAME, + password: DB_MASTER_PASSWORD, + database, + }); + await client.connect(); + return client; + }; + + const connectAsMaster = () => connectAs(DB_DEFAULT_DB); + + test("creates the slot when it is absent", async () => { + // Arrange + const event = buildCreateEvent(); + + // Act + await replicationSlotHandler(event); + + // Assert + const masterClient = await connectAsMaster(); + expect(await replicationSlotExists(masterClient, SLOT_NAME)).toBe(true); + await masterClient.end(); + }); + + test("leaves an existing slot in place", async () => { + // Arrange + await replicationSlotHandler(buildCreateEvent()); + + // Act + const response = await replicationSlotHandler(buildCreateEvent()); + + // Assert + expect(response).toEqual({ + PhysicalResourceId: expect.stringContaining("replication-slot-"), + }); + }); + + test("refuses to create the slot while the required publication is absent", async () => { + // Act + const createSlot = () => + replicationSlotHandler( + buildCreateEvent({ requiredPublication: "missing_pub" }) + ); + + // Assert + await expect(createSlot()).rejects.toThrow( + /publication "missing_pub" does not exist/ + ); + }); + + test("creates the slot once the required publication exists", async () => { + // Arrange + const masterClient = await connectAsMaster(); + await masterClient.query("CREATE TABLE outbox (id int primary key)"); + await masterClient.query( + "CREATE PUBLICATION required_pub FOR TABLE outbox WITH (publish = 'insert')" + ); + + // Act + await replicationSlotHandler( + buildCreateEvent({ requiredPublication: "required_pub" }) + ); + + // Assert + expect(await replicationSlotExists(masterClient, SLOT_NAME)).toBe(true); + await masterClient.end(); + }); + + test("drops the slot on delete", async () => { + // Arrange + const physicalResourceId = await createSlotAndReturnPhysicalId(); + + // Act + await replicationSlotHandler(buildDeleteEvent(physicalResourceId)); + + // Assert + const masterClient = await connectAsMaster(); + expect(await replicationSlotExists(masterClient, SLOT_NAME)).toBe(false); + await masterClient.end(); + }); + + test("refuses to change the plugin of an existing slot", async () => { + // Arrange + const physicalResourceId = await createSlotAndReturnPhysicalId(); + + // Act + const changePlugin = () => + replicationSlotHandler( + buildUpdateEvent({ physicalResourceId, plugin: "test_decoding" }) + ); + + // Assert + await expect(changePlugin()).rejects.toThrow(/cannot change in place/); + }); + + test("creates the new slot when the name changes", async () => { + // Arrange + const physicalResourceId = await createSlotAndReturnPhysicalId(); + const renamedSlot = "test_slot_renamed"; + + // Act + const updateResponse = await replicationSlotHandler( + buildUpdateEvent({ physicalResourceId, name: renamedSlot }) + ); + + // Assert: a changed physical id is what makes CloudFormation delete the + // resource holding the old slot after the new one is created + expect( + (updateResponse as { PhysicalResourceId: string }).PhysicalResourceId + ).not.toEqual(physicalResourceId); + const masterClient = await connectAsMaster(); + expect(await replicationSlotExists(masterClient, renamedSlot)).toBe(true); + await masterClient.end(); + }); + + test("refuses to adopt an existing slot created with another plugin", async () => { + // Arrange + const masterClient = await connectAsMaster(); + await masterClient.query( + "SELECT pg_create_logical_replication_slot($1, $2)", + [SLOT_NAME, "test_decoding"] + ); + + // Act + const adoptSlot = () => replicationSlotHandler(buildCreateEvent()); + + // Assert + await expect(adoptSlot()).rejects.toThrow( + /already exists in database "postgres" with plugin "test_decoding"/ + ); + await masterClient.end(); + }); + + test("refuses to move an existing slot to another database", async () => { + // Arrange + const masterClient = await connectAsMaster(); + await masterClient.query(`CREATE DATABASE ${OTHER_DB}`); + const physicalResourceId = await createSlotAndReturnPhysicalId(); + + // Act + const moveSlot = () => + replicationSlotHandler( + buildUpdateEvent({ physicalResourceId, database: OTHER_DB }) + ); + + // Assert: a slot name is unique across the cluster, so failing to create + // the replacement is what proves the changed database produced a + // new physical id instead of the update silently reporting success + await expect(moveSlot()).rejects.toThrow( + /replication slot "test_slot" already exists/ + ); + await masterClient.end(); + }); + + test("leaves a slot decoding another database in place on delete", async () => { + // Arrange + const masterClient = await connectAsMaster(); + await masterClient.query(`CREATE DATABASE ${OTHER_DB}`); + const otherClient = await connectAs(OTHER_DB); + await otherClient.query("SELECT pg_create_logical_replication_slot($1, $2)", [ + SLOT_NAME, + "pgoutput", + ]); + + // Act + await replicationSlotHandler(buildDeleteEvent(CREATED_PHYSICAL_ID)); + + // Assert + expect(await replicationSlotExists(otherClient, SLOT_NAME)).toBe(true); + await otherClient.end(); + await masterClient.end(); + }); + + test("leaves an adopted slot in place on delete", async () => { + // Arrange + const masterClient = await connectAsMaster(); + await masterClient.query( + "SELECT pg_create_logical_replication_slot($1, $2)", + [SLOT_NAME, "pgoutput"] + ); + + // Act + const physicalResourceId = await createSlotAndReturnPhysicalId(); + await replicationSlotHandler(buildDeleteEvent(physicalResourceId)); + + // Assert + expect(physicalResourceId).toMatch(/-adopted$/); + expect(await replicationSlotExists(masterClient, SLOT_NAME)).toBe(true); + await masterClient.end(); + }); + + test("leaves the slot in place when the physical id carries no provenance", async () => { + // Arrange + const masterClient = await connectAsMaster(); + await masterClient.query( + "SELECT pg_create_logical_replication_slot($1, $2)", + [SLOT_NAME, "pgoutput"] + ); + + // Act + await replicationSlotHandler( + buildDeleteEvent(PHYSICAL_ID_WITHOUT_PROVENANCE) + ); + + // Assert: rollback of a failed create deletes under a generated token, and + // a create fails precisely when a slot it never created is already + // there, so an id without provenance must never drop + expect(await replicationSlotExists(masterClient, SLOT_NAME)).toBe(true); + await masterClient.end(); + }); + + test("keeps the physical id when an adopted slot is updated with no change", async () => { + // Arrange + const masterClient = await connectAsMaster(); + await masterClient.query( + "SELECT pg_create_logical_replication_slot($1, $2)", + [SLOT_NAME, "pgoutput"] + ); + const physicalResourceId = await createSlotAndReturnPhysicalId(); + + // Act + const updateResponse = await replicationSlotHandler( + buildUpdateEvent({ physicalResourceId }) + ); + + // Assert + expect( + (updateResponse as { PhysicalResourceId: string }).PhysicalResourceId + ).toEqual(physicalResourceId); + await masterClient.end(); + }); +});