diff --git a/docs/publishing-migration.md b/docs/publishing-migration.md deleted file mode 100644 index 70e17aff..00000000 --- a/docs/publishing-migration.md +++ /dev/null @@ -1,60 +0,0 @@ -# Publication manifest migration - -`publicationMigrations` backfills data required by the atomic publication -path. It is resumable and supports `dryRun` and `apply` modes. - -The migration processes these phases in order: - -1. Add `searchText` to historical content revisions. -2. Add bounded `description` values to release pages. -3. Capture immutable `extractedText` on historical release files. -4. Add `releaseId` to legacy release search entries. -5. Rebuild the live search scope for every currently live site. - -The migration does not delete legacy fields or old per-release search entries. -Those are removed only after every environment has been migrated and verified. - -## Run a preview migration - -Run these commands from `packages/backend` against the Convex deployment used by -the preview deployment. Use a unique run ID for each mode. - -```sh -bunx convex run --deployment publicationMigrations:runBatch \ - '{"runId":"publication-preview-dry-YYYYMMDD","mode":"dryRun"}' - -bunx convex run --deployment publicationMigrations:getReport \ - '{"runId":"publication-preview-dry-YYYYMMDD"}' -``` - -Wait for `status` to become `completed`. Apply only when the dry run reports -`errorCount: 0`. `migratedCount` counts records that need backfilling; -`scheduledCount` counts live-search rebuilds that will be scheduled. - -```sh -bunx convex run --deployment publicationMigrations:runBatch \ - '{"runId":"publication-preview-apply-YYYYMMDD","mode":"apply"}' - -bunx convex run --deployment publicationMigrations:getReport \ - '{"runId":"publication-preview-apply-YYYYMMDD"}' -``` - -The batch function schedules its next batch automatically. A repeated command -with the same completed run ID is an idempotent no-op. - -After the apply run completes, verify that revisions, release pages, release -files, and release search entries have the new fields. Also wait for every -scheduled live-search projection to finish before testing published search. - -## Production gate - -Before applying in production: - -1. Take and verify a Convex backup. -2. Run the production dry run. -3. Investigate every migration error. -4. Record the dry-run report. -5. Apply the migration only after the preview verification passes. - -Do not remove compatibility code until all environments have completed the -backfill and the old record shapes have been separately audited. diff --git a/packages/backend/convex/_generated/api.d.ts b/packages/backend/convex/_generated/api.d.ts index e25e8772..0d5f3dcf 100644 --- a/packages/backend/convex/_generated/api.d.ts +++ b/packages/backend/convex/_generated/api.d.ts @@ -66,10 +66,7 @@ import type * as pageGuests from "../pageGuests.js"; import type * as pages from "../pages.js"; import type * as permissions from "../permissions.js"; import type * as publication from "../publication.js"; -import type * as publicationCleanup from "../publicationCleanup.js"; -import type * as publicationMigrations from "../publicationMigrations.js"; import type * as published from "../published.js"; -import type * as releasePublication from "../releasePublication.js"; import type * as releases from "../releases.js"; import type * as schema_aiCredits from "../schema/aiCredits.js"; import type * as schema_billing from "../schema/billing.js"; @@ -158,10 +155,7 @@ declare const fullApi: ApiFromModules<{ pages: typeof pages; permissions: typeof permissions; publication: typeof publication; - publicationCleanup: typeof publicationCleanup; - publicationMigrations: typeof publicationMigrations; published: typeof published; - releasePublication: typeof releasePublication; releases: typeof releases; "schema/aiCredits": typeof schema_aiCredits; "schema/billing": typeof schema_billing; diff --git a/packages/backend/convex/draftRestores.ts b/packages/backend/convex/draftRestores.ts index c2877fd5..cbc57c39 100644 --- a/packages/backend/convex/draftRestores.ts +++ b/packages/backend/convex/draftRestores.ts @@ -10,7 +10,6 @@ import { components, internal } from "./_generated/api"; import type { Doc } from "./_generated/dataModel"; import { mutation, query } from "./_generated/server"; import type { QueryCtx } from "./_generated/server"; -import { isReleaseAvailable } from "./model/releaseState"; import { isOrganizationMember, requireOrganizationPermission, @@ -29,9 +28,6 @@ export const restore = mutation({ site.organizationId, { resource: "content", action: "edit" }, ); - if (!isReleaseAvailable(release)) { - throw new ConvexError("Release is not available for restore"); - } if (site.activeDraftRestoreId) { const active = await ctx.db.get(site.activeDraftRestoreId); if ( diff --git a/packages/backend/convex/files.ts b/packages/backend/convex/files.ts index b47d1a33..8761308b 100644 --- a/packages/backend/convex/files.ts +++ b/packages/backend/convex/files.ts @@ -25,7 +25,6 @@ import { } from "./search"; import { recordStorageUsageEvent } from "./model/storageTelemetry"; import { pendingSiteAssetLifecycle } from "./model/siteAssets"; -import { isReleaseAvailable } from "./model/releaseState"; async function isFileReferencedByAccessiblePage( ctx: Parameters[0], @@ -180,7 +179,7 @@ export const getPublic = query({ const site = await ctx.db.get(file.siteId); if (!site?.liveReleaseId || !isPubliclyPublishedSite(site)) return null; const release = await ctx.db.get(site.liveReleaseId); - if (!release || !isReleaseAvailable(release)) return null; + if (!release) return null; const snapshot = await ctx.db .query("releaseFiles") .withIndex("by_release_file", (q) => @@ -212,7 +211,6 @@ export const getAuthorized = query({ : null; const released = liveRelease && - isReleaseAvailable(liveRelease) && (await ctx.db .query("releaseFiles") .withIndex("by_release_file", (q) => diff --git a/packages/backend/convex/libraries.ts b/packages/backend/convex/libraries.ts index d097e740..679af836 100644 --- a/packages/backend/convex/libraries.ts +++ b/packages/backend/convex/libraries.ts @@ -14,7 +14,6 @@ import { requireLibraryManagement, } from "./model/libraryAccess"; import { assertDraftReadable, touchSiteDraft } from "./model/draft"; -import { isReleaseAvailable } from "./model/releaseState"; const librarySummary = v.object({ _id: v.id("documentLibraries"), @@ -251,7 +250,7 @@ export const getPublishedExplorer = query({ const access = await resolvePublishedSiteAccess(ctx, site); if (!canRenderPublishedSite(access)) return null; const release = await ctx.db.get(site.liveReleaseId); - if (!release || !isReleaseAvailable(release)) return null; + if (!release) return null; const releasedLibrary = await ctx.db .query("releaseLibraries") .withIndex("by_release_library", (q) => diff --git a/packages/backend/convex/model/contentObjects.test.ts b/packages/backend/convex/model/contentObjects.test.ts index 821dea4a..2dc572de 100644 --- a/packages/backend/convex/model/contentObjects.test.ts +++ b/packages/backend/convex/model/contentObjects.test.ts @@ -2,41 +2,21 @@ import { describe, expect, test } from "bun:test"; import { readContentRevisionSearchText } from "./contentObjects"; describe("content revision search text", () => { - test("rebuilds missing search text for revisions written before the field existed", async () => { + test("reads the captured search text from a content revision", async () => { const revision = { _id: "revision-1", - payloadId: "payload-1", - searchText: undefined, - }; - const payload = { - _id: "payload-1", - content: { - type: "doc", - version: 1, - content: [ - { - type: "paragraph", - attrs: { "openeditor-id": "paragraph-1" }, - content: [{ type: "text", text: "Legacy searchable content" }], - }, - ], - }, + searchText: "Captured searchable content", }; const result = await readContentRevisionSearchText( { db: { - get: async (id: string) => - id === revision._id - ? revision - : id === payload._id - ? payload - : null, + get: async (id: string) => (id === revision._id ? revision : null), }, } as never, revision._id as never, ); - expect(result).toContain("Legacy searchable content"); + expect(result).toBe("Captured searchable content"); }); }); diff --git a/packages/backend/convex/model/contentObjects.ts b/packages/backend/convex/model/contentObjects.ts index 97f1bff0..decdde6c 100644 --- a/packages/backend/convex/model/contentObjects.ts +++ b/packages/backend/convex/model/contentObjects.ts @@ -3,17 +3,13 @@ import type { DataModel, Id } from "../_generated/dataModel"; import { extractOpenEditorReferences, extractOpenEditorText, - parseOpenEditorDocument, type OpenEditorDocument, } from "../pageContentFormat"; import { recordStorageUsageEvent } from "./storageTelemetry"; type MutationCtx = Pick, "db">; -/** - * Reads the denormalized text when present and rebuilds it for content - * revisions written before `searchText` was added to the schema. - */ +/** Reads the denormalized text captured on the content revision. */ export async function readContentRevisionSearchText( ctx: MutationCtx, revisionId: Id<"contentRevisions"> | undefined, @@ -21,16 +17,7 @@ export async function readContentRevisionSearchText( if (!revisionId) return ""; const revision = await ctx.db.get(revisionId); if (!revision) return ""; - if (revision.searchText !== undefined) return revision.searchText; - const payload = await ctx.db.get(revision.payloadId); - if (!payload) return ""; - try { - return extractOpenEditorText(parseOpenEditorDocument(payload.content)); - } catch { - // Search is a derived projection. Omit malformed historical content here; - // publication validates legacy payloads before activation. - return ""; - } + return revision.searchText; } export async function getOrCreateContentObject( diff --git a/packages/backend/convex/model/draftSummary.ts b/packages/backend/convex/model/draftSummary.ts index af33526d..e42469fd 100644 --- a/packages/backend/convex/model/draftSummary.ts +++ b/packages/backend/convex/model/draftSummary.ts @@ -1,16 +1,12 @@ import type { Doc } from "../_generated/dataModel"; import type { QueryCtx } from "../_generated/server"; -import { isReleaseAvailable } from "./releaseState"; export async function buildDraftSummary(ctx: QueryCtx, site: Doc<"sites">) { const liveReleaseCandidate = site.liveReleaseId ? await ctx.db.get(site.liveReleaseId) : null; const liveRelease = - liveReleaseCandidate?.siteId === site._id && - isReleaseAvailable(liveReleaseCandidate) - ? liveReleaseCandidate - : null; + liveReleaseCandidate?.siteId === site._id ? liveReleaseCandidate : null; const hasDraftChanges = (await ctx.db .query("draftChanges") diff --git a/packages/backend/convex/model/publishedRelease.ts b/packages/backend/convex/model/publishedRelease.ts index 57212109..ef0fe03c 100644 --- a/packages/backend/convex/model/publishedRelease.ts +++ b/packages/backend/convex/model/publishedRelease.ts @@ -1,6 +1,5 @@ import type { Doc, Id } from "../_generated/dataModel"; import type { QueryCtx } from "../_generated/server"; -import { isReleaseAvailable } from "./releaseState"; import { canRenderPublishedSite, resolvePublishedSiteAccess } from "../sharing"; export type ResolvedReleasePage = { @@ -101,7 +100,7 @@ export async function getReadableLiveRelease( releaseId: Id<"siteReleases">, ) { const release = await ctx.db.get(releaseId); - if (!release || !isReleaseAvailable(release)) return null; + if (!release) return null; const site = await ctx.db.get(release.siteId); if (!site || site.liveReleaseId !== release._id) return null; const access = await resolvePublishedSiteAccess(ctx, site); diff --git a/packages/backend/convex/model/releaseState.test.ts b/packages/backend/convex/model/releaseState.test.ts index 8422e5e1..e5bda40e 100644 --- a/packages/backend/convex/model/releaseState.test.ts +++ b/packages/backend/convex/model/releaseState.test.ts @@ -2,7 +2,6 @@ import { describe, expect, test } from "bun:test"; import { extractionBlocksPublication, extractionRetryInvalidatesDraft, - isReleaseAvailable, publicationActionForTarget, } from "./releaseState"; @@ -34,12 +33,4 @@ describe("release publication guards", () => { expect(extractionRetryInvalidatesDraft(undefined)).toBe(true); expect(extractionRetryInvalidatesDraft("processing")).toBe(false); }); - - test("keeps atomic and completed legacy releases readable", () => { - expect(isReleaseAvailable({})).toBe(true); - expect(isReleaseAvailable({ publicationStatus: "complete" })).toBe(true); - expect(isReleaseAvailable({ publicationStatus: "clearing" })).toBe(true); - expect(isReleaseAvailable({ publicationStatus: "building" })).toBe(false); - expect(isReleaseAvailable({ publicationStatus: "failed" })).toBe(false); - }); }); diff --git a/packages/backend/convex/model/releaseState.ts b/packages/backend/convex/model/releaseState.ts index 50697c13..924faf95 100644 --- a/packages/backend/convex/model/releaseState.ts +++ b/packages/backend/convex/model/releaseState.ts @@ -1,5 +1,3 @@ -import type { Doc } from "../_generated/dataModel"; - export function publicationActionForTarget( currentNumber: number | undefined, targetNumber: number, @@ -9,15 +7,6 @@ export function publicationActionForTarget( : "republish"; } -export type PublicationStatus = "building" | "clearing" | "complete" | "failed"; - -/** Compatibility state helpers for releases created by the retired workflow. */ -export function isPublicationInFlight( - status: PublicationStatus | undefined, -): boolean { - return status === "building" || status === "clearing"; -} - export function extractionBlocksPublication( status: "queued" | "processing" | "ready" | "failed", ): boolean { @@ -45,17 +34,3 @@ export function extractionIsPublishable( (extraction.status === "ready" || extraction.status === "failed") ); } - -/** - * Old releases may still carry a publication workflow status. New releases - * activate atomically and have no status, so an absent status is readable. - */ -export function isReleaseAvailable( - release: Pick, "publicationStatus">, -): boolean { - return ( - release.publicationStatus === undefined || - release.publicationStatus === "complete" || - release.publicationStatus === "clearing" - ); -} diff --git a/packages/backend/convex/publication.test.ts b/packages/backend/convex/publication.test.ts index 63d6a6f4..301a0d7b 100644 --- a/packages/backend/convex/publication.test.ts +++ b/packages/backend/convex/publication.test.ts @@ -367,7 +367,7 @@ describe("publish", () => { expect(state.inserted).toHaveLength(0); }); - test("rejects malformed legacy page content before activation", async () => { + test("rejects a page with a missing captured revision", async () => { const site: Row = { _id: "site-1", organizationId: "organization-1", @@ -394,27 +394,10 @@ describe("publish", () => { contentHash: "hash-1", updatedAt: 1, }; - const revision: Row = { - _id: "revision-1", - siteId: site._id, - payloadId: "payload-1", - }; const state = makeDb({ sites: [site], pages: [page], pageDocuments: [document], - contentRevisions: [revision], - contentPayloads: [ - { - _id: "payload-1", - siteId: site._id, - content: JSON.stringify({ - type: "doc", - version: 1, - content: [{ type: "unknownHistoricalNode" }], - }), - }, - ], draftChanges: [], }); @@ -507,7 +490,7 @@ describe("live search projection", () => { ); }); - test("does not leak current extraction into a legacy release", async () => { + test("uses empty text when a file snapshot has no captured text", async () => { const site: Row = { _id: "site-1", liveReleaseId: "release-1", diff --git a/packages/backend/convex/publication.ts b/packages/backend/convex/publication.ts index fd5c6eb6..19ad2320 100644 --- a/packages/backend/convex/publication.ts +++ b/packages/backend/convex/publication.ts @@ -5,14 +5,7 @@ import { internalMutation, type MutationCtx } from "./_generated/server"; import { buildReleaseChangeDetail } from "./model/releaseChangeDetails"; import { fileSourceVersion } from "./model/fileExtraction"; import { readContentRevisionSearchText } from "./model/contentObjects"; -import { - extractionIsPublishable, - isReleaseAvailable, -} from "./model/releaseState"; -import { - extractOpenEditorText, - parseOpenEditorDocument, -} from "./pageContentFormat"; +import { extractionIsPublishable } from "./model/releaseState"; import { upsertSearchEntry } from "./search"; /** @@ -44,17 +37,7 @@ async function readPublishablePageText( if (!revision || revision.siteId !== siteId) { throw new Error("Page content revision is missing"); } - // New revisions already carry their denormalized text. Avoid loading the - // potentially large payload in the publish transaction just to derive the - // bounded description below. - if (revision.searchText !== undefined) return revision.searchText; - const payload = await ctx.db.get(revision.payloadId); - if (!payload || payload.siteId !== siteId) { - throw new Error("Page content payload is missing"); - } - // Parse legacy revisions once so malformed content cannot become the live - // release. - return extractOpenEditorText(parseOpenEditorDocument(payload.content)); + return revision.searchText; } export function truncateDescription(text: string): string { @@ -241,15 +224,12 @@ export async function snapshotFiles( order: source.order, uploadedBy: source.uploadedBy, createdAt: source.createdAt, - ...(source.kind === "file" - ? { - extractedText: - extraction?.status === "ready" && - extraction.sourceVersion === fileSourceVersion(source) - ? (extraction.extractedText ?? "") - : "", - } - : {}), + extractedText: + source.kind === "file" && + extraction?.status === "ready" && + extraction.sourceVersion === fileSourceVersion(source) + ? (extraction.extractedText ?? "") + : "", }); } } @@ -299,35 +279,8 @@ export async function clearPublishedDraftChanges( } } -async function readReleasedFileText( - ctx: MutationCtx, - row: Doc<"releaseFiles">, -): Promise { - if (row.extractedText !== undefined) return row.extractedText; - - // Releases created before `releaseFiles.extractedText` was added need a - // guarded compatibility fallback. Never use an extraction for a different - // file source; doing so would make historical releases search current draft - // content. - const file = await ctx.db.get(row.fileId); - if ( - file?.kind !== "file" || - file.deletedAt !== undefined || - file.objectKey !== row.objectKey || - file.size !== row.size || - file.checksum !== row.checksum - ) { - return ""; - } - - const extraction = await ctx.db - .query("fileExtractions") - .withIndex("by_file", (q) => q.eq("fileId", row.fileId)) - .unique(); - return extraction?.status === "ready" && - extraction.sourceVersion === fileSourceVersion(file) - ? (extraction.extractedText ?? "") - : ""; +async function readReleasedFileText(row: Doc<"releaseFiles">): Promise { + return row.extractedText ?? ""; } async function scheduleLiveSearchBatch( @@ -426,11 +379,7 @@ export const projectLiveSearchBatch = internalMutation({ } if (!expectedLiveReleaseId) return null; const release = await ctx.db.get(expectedLiveReleaseId); - if ( - !release || - release.siteId !== siteId || - !isReleaseAvailable(release) - ) { + if (!release || release.siteId !== siteId) { return null; } await scheduleLiveSearchBatch(ctx, { @@ -444,7 +393,7 @@ export const projectLiveSearchBatch = internalMutation({ if (!expectedLiveReleaseId) return null; const release = await ctx.db.get(expectedLiveReleaseId); - if (!release || release.siteId !== siteId || !isReleaseAvailable(release)) { + if (!release || release.siteId !== siteId) { return null; } @@ -487,7 +436,7 @@ export const projectLiveSearchBatch = internalMutation({ kind: "file", sourceId: row.fileId, title: row.filename, - text: await readReleasedFileText(ctx, row), + text: await readReleasedFileText(row), }); } if (!page.isDone) { diff --git a/packages/backend/convex/publicationCleanup.ts b/packages/backend/convex/publicationCleanup.ts deleted file mode 100644 index 28da9688..00000000 --- a/packages/backend/convex/publicationCleanup.ts +++ /dev/null @@ -1,216 +0,0 @@ -import { - cleanup as cleanupWorkflow, - getStatus, - type WorkflowId, - type WorkflowStatus, -} from "@convex-dev/workflow"; -import { v } from "convex/values"; -import { components, internal } from "./_generated/api"; -import type { Id } from "./_generated/dataModel"; -import type { MutationCtx } from "./_generated/server"; -import { internalMutation } from "./_generated/server"; - -const CLEANUP_BATCH_SIZE = 25; - -const cleanupPhase = v.union( - v.literal("releases"), - v.literal("pages"), - v.literal("search"), - v.literal("migrationRuns"), -); - -type CleanupPhase = "releases" | "pages" | "search" | "migrationRuns"; - -const nextPhase: Record = { - releases: "pages", - pages: "search", - search: "migrationRuns", - migrationRuns: undefined, -}; - -async function scheduleNext( - ctx: Pick, - args: { phase: CleanupPhase; cursor?: string }, -) { - await ctx.scheduler.runAfter(0, internal.publicationCleanup.run, args); -} - -async function assertReleaseHasNoReferences( - ctx: MutationCtx, - releaseId: Id<"siteReleases">, - siteId: Id<"sites">, -): Promise { - const siteReference = await ctx.db - .query("sites") - .filter((q) => - q.or( - q.eq(q.field("liveReleaseId"), releaseId), - q.eq(q.field("draftBaseReleaseId"), releaseId), - ), - ) - .first(); - const successor = await ctx.db - .query("siteReleases") - .filter((q) => q.eq(q.field("previousReleaseId"), releaseId)) - .first(); - const [page, library, folder, file, change] = await Promise.all([ - ctx.db - .query("releasePages") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .first(), - ctx.db - .query("releaseLibraries") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .first(), - ctx.db - .query("releaseFolders") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .first(), - ctx.db - .query("releaseFiles") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .first(), - ctx.db - .query("releaseChanges") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .first(), - ]); - const event = (await ctx.db - .query("publicationEvents") - .withIndex("by_site", (q) => q.eq("siteId", siteId)) - .filter((q) => - q.or( - q.eq(q.field("fromReleaseId"), releaseId), - q.eq(q.field("toReleaseId"), releaseId), - ), - ) - .first()) as unknown; - - if ( - siteReference || - successor || - page || - library || - folder || - file || - change || - event - ) { - throw new Error(`Failed release ${releaseId} still has references`); - } -} - -async function cleanupWorkflowStorage( - ctx: MutationCtx, - workflowId: string, -): Promise { - const typedWorkflowId = workflowId as WorkflowId; - let status: WorkflowStatus; - try { - status = await getStatus(ctx, components.workflow, typedWorkflowId); - } catch (error) { - if (error instanceof Error && error.message.includes("not found")) { - return; - } - throw error; - } - if (status.type === "inProgress") { - throw new Error( - `Legacy publication workflow ${workflowId} is still active`, - ); - } - await cleanupWorkflow(ctx, components.workflow, typedWorkflowId); -} - -export const run = internalMutation({ - args: { - phase: cleanupPhase, - cursor: v.optional(v.string()), - }, - returns: v.null(), - handler: async (ctx, { phase, cursor }) => { - if (phase === "releases") { - const page = await ctx.db - .query("siteReleases") - .paginate({ cursor: cursor ?? null, numItems: CLEANUP_BATCH_SIZE }); - for (const release of page.page) { - if (release.publicationStatus === "failed") { - await assertReleaseHasNoReferences(ctx, release._id, release.siteId); - if (release.publicationWorkflowId) { - await cleanupWorkflowStorage(ctx, release.publicationWorkflowId); - } - await ctx.db.delete(release._id); - continue; - } - if ( - release.publicationStatus !== undefined || - release.publicationFailure !== undefined || - release.publicationWorkflowId !== undefined || - release.publicationUpdatedAt !== undefined - ) { - if (release.publicationWorkflowId) { - await cleanupWorkflowStorage(ctx, release.publicationWorkflowId); - } - await ctx.db.patch(release._id, { - publicationStatus: undefined, - publicationFailure: undefined, - publicationWorkflowId: undefined, - publicationUpdatedAt: undefined, - }); - } - } - if (!page.isDone) { - await scheduleNext(ctx, { phase, cursor: page.continueCursor }); - } else { - await scheduleNext(ctx, { phase: nextPhase[phase]! }); - } - return null; - } - - if (phase === "pages") { - const page = await ctx.db - .query("releasePages") - .paginate({ cursor: cursor ?? null, numItems: CLEANUP_BATCH_SIZE }); - for (const releasePage of page.page) { - if (releasePage.descriptionText !== undefined) { - await ctx.db.patch(releasePage._id, { descriptionText: undefined }); - } - } - if (!page.isDone) { - await scheduleNext(ctx, { phase, cursor: page.continueCursor }); - } else { - await scheduleNext(ctx, { phase: nextPhase[phase]! }); - } - return null; - } - - if (phase === "search") { - const page = await ctx.db - .query("searchEntries") - .paginate({ cursor: cursor ?? null, numItems: CLEANUP_BATCH_SIZE }); - for (const entry of page.page) { - if ( - entry.scopeId.startsWith("release:") || - entry.scopeId.startsWith("site:") - ) { - await ctx.db.delete(entry._id); - } - } - if (!page.isDone) { - await scheduleNext(ctx, { phase, cursor: page.continueCursor }); - } else { - await scheduleNext(ctx, { phase: nextPhase[phase]! }); - } - return null; - } - - const page = await ctx.db - .query("publicationMigrationRuns") - .paginate({ cursor: cursor ?? null, numItems: CLEANUP_BATCH_SIZE }); - for (const run of page.page) await ctx.db.delete(run._id); - if (!page.isDone) { - await scheduleNext(ctx, { phase, cursor: page.continueCursor }); - } - return null; - }, -}); diff --git a/packages/backend/convex/publicationMigrations.test.ts b/packages/backend/convex/publicationMigrations.test.ts deleted file mode 100644 index 83824e4f..00000000 --- a/packages/backend/convex/publicationMigrations.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { internal } from "./_generated/api"; -import { runBatch } from "./publicationMigrations"; - -type RegisteredFunction = { - _handler: (ctx: unknown, args: unknown) => Promise; -}; - -type Row = Record & { _id: string }; - -function invoke(fn: unknown, ctx: unknown, args: unknown): Promise { - return (fn as RegisteredFunction)._handler(ctx, args); -} - -function makeContext(tables: Record) { - const scheduled: Array<{ - functionReference: unknown; - args: Record; - }> = []; - let nextId = 1; - - const findRows = (table: string, constraints: Array<[string, unknown]>) => - (tables[table] ?? []).filter((row) => - constraints.every(([field, value]) => row[field] === value), - ); - - const query = (table: string, constraints: Array<[string, unknown]> = []) => { - const rows = () => findRows(table, constraints); - const builder = { - unique: async () => { - const matches = rows(); - if (matches.length > 1) throw new Error("expected unique"); - return matches[0] ?? null; - }, - paginate: async ({ - cursor, - numItems, - }: { - cursor: string | null; - numItems: number; - }) => { - const offset = cursor ? Number(cursor) : 0; - const page = rows().slice(offset, offset + numItems); - return { - page, - isDone: offset + page.length >= rows().length, - continueCursor: String(offset + page.length), - }; - }, - }; - return { - ...builder, - withIndex: ( - _indexName: string, - resolve: (q: { - eq: (field: string, value: unknown) => unknown; - }) => unknown, - ) => { - const indexConstraints: Array<[string, unknown]> = []; - const q = { - eq: (field: string, value: unknown) => { - indexConstraints.push([field, value]); - return q; - }, - }; - resolve(q); - return query(table, indexConstraints); - }, - }; - }; - - const db = { - get: async (id: string) => { - for (const rows of Object.values(tables)) { - const row = rows.find((candidate) => candidate._id === id); - if (row) return row; - } - return null; - }, - insert: async (table: string, value: Record) => { - const id = `${table}-${nextId++}`; - const rows = tables[table] ?? []; - rows.push({ ...value, _id: id }); - tables[table] = rows; - return id; - }, - patch: async (id: string, value: Record) => { - for (const rows of Object.values(tables)) { - const row = rows.find((candidate) => candidate._id === id); - if (row) Object.assign(row, value); - } - }, - query, - normalizeId: (table: string, id: string) => - (tables[table] ?? []).some((row) => row._id === id) ? id : null, - }; - - const ctx = { - db, - scheduler: { - runAfter: async ( - _delay: number, - functionReference: unknown, - args: Record, - ) => { - scheduled.push({ functionReference, args }); - }, - }, - }; - return { ctx, scheduled, tables }; -} - -function migrationFixture() { - return makeContext({ - sites: [ - { - _id: "site-1", - liveReleaseId: "release-1", - organizationId: "organization-1", - }, - ], - siteReleases: [{ _id: "release-1", siteId: "site-1", number: 1 }], - contentRevisions: [ - { - _id: "revision-1", - siteId: "site-1", - payloadId: "payload-1", - fileIds: [], - libraryIds: [], - pageIds: ["page-1"], - }, - ], - contentPayloads: [ - { - _id: "payload-1", - siteId: "site-1", - content: JSON.stringify({ - type: "doc", - version: 1, - content: [ - { - type: "paragraph", - attrs: { "openeditor-id": "paragraph-1" }, - content: [{ type: "text", text: "Migrated body" }], - }, - ], - }), - }, - ], - releasePages: [ - { - _id: "release-page-1", - releaseId: "release-1", - siteId: "site-1", - pageId: "page-1", - contentRevisionId: "revision-1", - descriptionText: "Legacy description", - }, - ], - releaseFiles: [ - { - _id: "release-file-1", - releaseId: "release-1", - siteId: "site-1", - fileId: "file-1", - kind: "file", - objectKey: "documents/file-1", - size: 10, - checksum: "checksum-1", - }, - ], - searchEntries: [ - { - _id: "search-page-1", - siteId: "site-1", - scopeId: "release:release-1", - kind: "page", - sourceId: "page-1", - title: "Home", - text: "Migrated body", - }, - { - _id: "search-file-1", - siteId: "site-1", - scopeId: "release:release-1", - kind: "file", - sourceId: "file-1", - title: "guide.md", - text: "Historical file text", - }, - ], - }); -} - -async function drainMigration( - state: ReturnType, - args: { runId: string; mode: "dryRun" | "apply" }, -) { - await invoke(runBatch, state.ctx, args); - while (true) { - const index = state.scheduled.findIndex( - (item) => item.args.runId === args.runId, - ); - if (index === -1) return; - const [next] = state.scheduled.splice(index, 1); - if (next) await invoke(runBatch, state.ctx, next.args); - } -} - -describe("publication migration", () => { - test("backfills immutable text, release metadata, search identity, and live projection", async () => { - const state = migrationFixture(); - - await drainMigration(state, { runId: "apply-1", mode: "apply" }); - - expect(state.tables.contentRevisions?.[0]?.searchText).toContain( - "Migrated body", - ); - expect(state.tables.releasePages?.[0]?.description).toContain( - "Migrated body", - ); - expect(state.tables.releaseFiles?.[0]?.extractedText).toBe( - "Historical file text", - ); - expect(state.tables.searchEntries).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - _id: "search-page-1", - releaseId: "release-1", - }), - expect.objectContaining({ - _id: "search-file-1", - releaseId: "release-1", - }), - ]), - ); - expect(state.tables.sites?.[0]?.liveSearchProjectionGeneration).toBe(1); - expect(state.tables.publicationMigrationRuns?.[0]).toMatchObject({ - status: "completed", - migratedCount: 6, - scheduledCount: 1, - skippedCount: 0, - errorCount: 0, - }); - expect(state.scheduled).toContainEqual({ - functionReference: internal.publication.projectLiveSearch, - args: { - siteId: "site-1", - expectedLiveReleaseId: "release-1", - expectedLiveSearchProjectionGeneration: 1, - }, - }); - }); - - test("dry run reports work without changing publication data", async () => { - const state = migrationFixture(); - - await drainMigration(state, { runId: "dry-1", mode: "dryRun" }); - - expect(state.tables.contentRevisions?.[0]?.searchText).toBeUndefined(); - expect(state.tables.releasePages?.[0]?.description).toBeUndefined(); - expect(state.tables.releaseFiles?.[0]?.extractedText).toBeUndefined(); - expect(state.tables.searchEntries?.[0]?.releaseId).toBeUndefined(); - expect( - state.tables.sites?.[0]?.liveSearchProjectionGeneration, - ).toBeUndefined(); - expect(state.tables.publicationMigrationRuns?.[0]).toMatchObject({ - status: "completed", - migratedCount: 6, - scheduledCount: 1, - skippedCount: 0, - errorCount: 0, - }); - expect( - state.scheduled.some( - (item) => - item.functionReference === internal.publication.projectLiveSearch, - ), - ).toBe(false); - }); - - test("persists a failure for malformed historical content", async () => { - const state = migrationFixture(); - const payload = state.tables.contentPayloads?.[0]; - if (payload) payload.content = "not-json"; - - await drainMigration(state, { runId: "bad-1", mode: "apply" }); - - expect(state.tables.publicationMigrationRuns?.[0]).toMatchObject({ - status: "failed", - errorCount: 1, - failureSummary: expect.stringContaining("Unexpected"), - }); - expect(state.scheduled).toEqual([]); - }); -}); diff --git a/packages/backend/convex/publicationMigrations.ts b/packages/backend/convex/publicationMigrations.ts deleted file mode 100644 index 1ac9e014..00000000 --- a/packages/backend/convex/publicationMigrations.ts +++ /dev/null @@ -1,408 +0,0 @@ -import { v } from "convex/values"; -import { internal } from "./_generated/api"; -import type { Doc } from "./_generated/dataModel"; -import { - internalMutation, - internalQuery, - type QueryCtx, - type MutationCtx, -} from "./_generated/server"; -import { - extractOpenEditorText, - parseOpenEditorDocument, -} from "./pageContentFormat"; -import { fileSourceVersion } from "./model/fileExtraction"; -import { readContentRevisionSearchText } from "./model/contentObjects"; -import { truncateDescription } from "./publication"; - -const PUBLICATION_MIGRATION_KEY = "publication-manifest-v1"; -const BATCH_SIZE = 8; - -type MigrationPhase = "revisions" | "pages" | "files" | "search" | "sites"; -type MigrationMode = "dryRun" | "apply"; -type MigrationRun = Doc<"publicationMigrationRuns">; - -function nextPhase(phase: MigrationPhase): MigrationPhase | undefined { - switch (phase) { - case "revisions": - return "pages"; - case "pages": - return "files"; - case "files": - return "search"; - case "search": - return "sites"; - case "sites": - return undefined; - } -} - -async function findRun( - ctx: Pick, - runId: string, -): Promise { - return await ctx.db - .query("publicationMigrationRuns") - .withIndex("by_migration_run", (q) => - q.eq("migrationKey", PUBLICATION_MIGRATION_KEY).eq("runId", runId), - ) - .unique(); -} - -type Counters = { - scannedCount: number; - migratedCount: number; - skippedCount: number; - scheduledCount: number; - errorCount: number; -}; - -type Page = { - page: T[]; - isDone: boolean; - continueCursor: string; -}; - -function advance( - run: MigrationRun, - page: { isDone: boolean; continueCursor: string }, - counters: Counters, - now: number, -) { - const phase = page.isDone ? nextPhase(run.phase) : run.phase; - const completed = page.isDone && phase === undefined; - return { - ...counters, - phase: phase ?? run.phase, - cursor: page.isDone ? undefined : page.continueCursor, - status: completed ? ("completed" as const) : ("running" as const), - ...(completed ? { completedAt: now } : {}), - }; -} - -function countScanned(counters: Counters, amount: number): Counters { - return { ...counters, scannedCount: counters.scannedCount + amount }; -} - -function countersFromRun(run: MigrationRun): Counters { - return { - scannedCount: run.scannedCount, - migratedCount: run.migratedCount, - skippedCount: run.skippedCount, - scheduledCount: run.scheduledCount ?? 0, - errorCount: run.errorCount, - }; -} - -async function migrateRevisions( - ctx: MutationCtx, - run: MigrationRun, -): Promise> { - const page = (await ctx.db - .query("contentRevisions") - .paginate({ cursor: run.cursor ?? null, numItems: BATCH_SIZE })) as Page< - Doc<"contentRevisions"> - >; - let counters = countScanned(countersFromRun(run), page.page.length); - for (const revision of page.page) { - if (revision.searchText !== undefined) { - counters = { ...counters, skippedCount: counters.skippedCount + 1 }; - continue; - } - const payload = await ctx.db.get(revision.payloadId); - if (!payload || payload.siteId !== revision.siteId) { - throw new Error( - `Content payload is missing for revision ${revision._id}`, - ); - } - const searchText = extractOpenEditorText( - parseOpenEditorDocument(payload.content), - ); - if (run.mode === "apply") { - await ctx.db.patch(revision._id, { searchText }); - } - counters = { ...counters, migratedCount: counters.migratedCount + 1 }; - } - return advance(run, page, counters, Date.now()); -} - -async function migrateReleasePages( - ctx: MutationCtx, - run: MigrationRun, -): Promise> { - const page = (await ctx.db - .query("releasePages") - .paginate({ cursor: run.cursor ?? null, numItems: BATCH_SIZE })) as Page< - Doc<"releasePages"> - >; - let counters = countScanned(countersFromRun(run), page.page.length); - for (const releasePage of page.page) { - if (releasePage.description !== undefined) { - counters = { ...counters, skippedCount: counters.skippedCount + 1 }; - continue; - } - const contentText = await readContentRevisionSearchText( - ctx, - releasePage.contentRevisionId, - ); - const description = truncateDescription( - contentText || releasePage.descriptionText || "", - ); - if (run.mode === "apply") { - await ctx.db.patch(releasePage._id, { description }); - } - counters = { ...counters, migratedCount: counters.migratedCount + 1 }; - } - return advance(run, page, counters, Date.now()); -} - -async function readHistoricalFileText( - ctx: MutationCtx, - releaseFile: Doc<"releaseFiles">, -): Promise { - const legacyEntry = await ctx.db - .query("searchEntries") - .withIndex("by_scope_source", (q) => - q - .eq("scopeId", `release:${releaseFile.releaseId}`) - .eq("kind", "file") - .eq("sourceId", releaseFile.fileId), - ) - .unique(); - if ( - legacyEntry && - legacyEntry.siteId === releaseFile.siteId && - (legacyEntry.releaseId === undefined || - legacyEntry.releaseId === releaseFile.releaseId) - ) { - return legacyEntry.text; - } - - const file = await ctx.db.get(releaseFile.fileId); - if ( - file?.siteId !== releaseFile.siteId || - file.kind !== "file" || - file.deletedAt !== undefined || - file.objectKey !== releaseFile.objectKey || - file.size !== releaseFile.size || - file.checksum !== releaseFile.checksum - ) { - return ""; - } - const extraction = await ctx.db - .query("fileExtractions") - .withIndex("by_file", (q) => q.eq("fileId", releaseFile.fileId)) - .unique(); - return extraction?.status === "ready" && - extraction.sourceVersion === fileSourceVersion(file) - ? (extraction.extractedText ?? "") - : ""; -} - -async function migrateReleaseFiles( - ctx: MutationCtx, - run: MigrationRun, -): Promise> { - const page = (await ctx.db - .query("releaseFiles") - .paginate({ cursor: run.cursor ?? null, numItems: BATCH_SIZE })) as Page< - Doc<"releaseFiles"> - >; - let counters = countScanned(countersFromRun(run), page.page.length); - for (const releaseFile of page.page) { - if ( - releaseFile.kind !== "file" || - releaseFile.extractedText !== undefined - ) { - counters = { ...counters, skippedCount: counters.skippedCount + 1 }; - continue; - } - const extractedText = await readHistoricalFileText(ctx, releaseFile); - if (run.mode === "apply") { - await ctx.db.patch(releaseFile._id, { extractedText }); - } - counters = { ...counters, migratedCount: counters.migratedCount + 1 }; - } - return advance(run, page, counters, Date.now()); -} - -async function migrateSearchEntries( - ctx: MutationCtx, - run: MigrationRun, -): Promise> { - const page = (await ctx.db - .query("searchEntries") - .paginate({ cursor: run.cursor ?? null, numItems: BATCH_SIZE })) as Page< - Doc<"searchEntries"> - >; - let counters = countScanned(countersFromRun(run), page.page.length); - for (const entry of page.page) { - if ( - !entry.scopeId.startsWith("release:") || - entry.releaseId !== undefined - ) { - counters = { ...counters, skippedCount: counters.skippedCount + 1 }; - continue; - } - const releaseId = ctx.db.normalizeId( - "siteReleases", - entry.scopeId.slice("release:".length), - ); - if (!releaseId) { - throw new Error( - `Release search entry points to a missing release: ${entry._id}`, - ); - } - const release = await ctx.db.get(releaseId); - if (!release || release.siteId !== entry.siteId) { - throw new Error(`Release search entry has the wrong site: ${entry._id}`); - } - if (run.mode === "apply") { - await ctx.db.patch(entry._id, { releaseId }); - } - counters = { ...counters, migratedCount: counters.migratedCount + 1 }; - } - return advance(run, page, counters, Date.now()); -} - -async function migrateLiveSites( - ctx: MutationCtx, - run: MigrationRun, -): Promise> { - const page = (await ctx.db - .query("sites") - .paginate({ cursor: run.cursor ?? null, numItems: BATCH_SIZE })) as Page< - Doc<"sites"> - >; - let counters = countScanned(countersFromRun(run), page.page.length); - for (const site of page.page) { - if (!site.liveReleaseId) { - counters = { ...counters, skippedCount: counters.skippedCount + 1 }; - continue; - } - const release = await ctx.db.get(site.liveReleaseId); - if (!release || release.siteId !== site._id) { - throw new Error(`Live release is missing for site ${site._id}`); - } - counters = { - ...counters, - scheduledCount: counters.scheduledCount + 1, - }; - if (run.mode === "apply") { - const generation = (site.liveSearchProjectionGeneration ?? 0) + 1; - await ctx.db.patch(site._id, { - liveSearchProjectionGeneration: generation, - }); - await ctx.scheduler.runAfter(0, internal.publication.projectLiveSearch, { - siteId: site._id, - expectedLiveReleaseId: site.liveReleaseId, - expectedLiveSearchProjectionGeneration: generation, - }); - counters = { - ...counters, - migratedCount: counters.migratedCount + 1, - }; - } else if (site.liveSearchProjectionGeneration === undefined) { - counters = { - ...counters, - migratedCount: counters.migratedCount + 1, - }; - } else { - counters = { ...counters, skippedCount: counters.skippedCount + 1 }; - } - } - return advance(run, page, counters, Date.now()); -} - -async function processPhase(ctx: MutationCtx, run: MigrationRun) { - switch (run.phase) { - case "revisions": - return await migrateRevisions(ctx, run); - case "pages": - return await migrateReleasePages(ctx, run); - case "files": - return await migrateReleaseFiles(ctx, run); - case "search": - return await migrateSearchEntries(ctx, run); - case "sites": - return await migrateLiveSites(ctx, run); - } -} - -async function createRun( - ctx: MutationCtx, - runId: string, - mode: MigrationMode, -): Promise { - const now = Date.now(); - const id = await ctx.db.insert("publicationMigrationRuns", { - migrationKey: PUBLICATION_MIGRATION_KEY, - runId, - mode, - phase: "revisions", - status: "running", - scannedCount: 0, - migratedCount: 0, - skippedCount: 0, - scheduledCount: 0, - errorCount: 0, - startedAt: now, - updatedAt: now, - }); - const run = await ctx.db.get(id); - if (!run) throw new Error("Publication migration run was not created"); - return run; -} - -export const runBatch = internalMutation({ - args: { - runId: v.string(), - mode: v.union(v.literal("dryRun"), v.literal("apply")), - }, - returns: v.any(), - handler: async (ctx, { runId, mode }) => { - const existing = await findRun(ctx, runId); - if (existing?.mode !== undefined && existing.mode !== mode) { - throw new Error("A migration run cannot change mode"); - } - if (existing?.status === "completed") return existing; - const run = existing ?? (await createRun(ctx, runId, mode)); - - try { - const update = await processPhase(ctx, run); - await ctx.db.patch(run._id, { - ...update, - failureSummary: undefined, - updatedAt: Date.now(), - }); - if (update.status === "running") { - await ctx.scheduler.runAfter( - 0, - internal.publicationMigrations.runBatch, - { - runId, - mode, - }, - ); - } - return await ctx.db.get(run._id); - } catch (error) { - const failureSummary = - error instanceof Error ? error.message : String(error); - await ctx.db.patch(run._id, { - status: "failed", - phase: run.phase, - errorCount: run.errorCount + 1, - failureSummary, - updatedAt: Date.now(), - }); - return await ctx.db.get(run._id); - } - }, -}); - -export const getReport = internalQuery({ - args: { runId: v.string() }, - returns: v.any(), - handler: async (ctx, { runId }) => await findRun(ctx, runId), -}); diff --git a/packages/backend/convex/published.ts b/packages/backend/convex/published.ts index ea9ba4db..2071bd2f 100644 --- a/packages/backend/convex/published.ts +++ b/packages/backend/convex/published.ts @@ -8,7 +8,6 @@ import { getReadableLiveRelease, resolveReleasePage, } from "./model/publishedRelease"; -import { isReleaseAvailable } from "./model/releaseState"; import { buildPageTree } from "./pages"; import { emptyOpenEditorDocument, @@ -90,7 +89,7 @@ async function getPublishedSiteBySlug( .unique(); if (!site?.liveReleaseId) return null; const release = await ctx.db.get(site.liveReleaseId); - return release && isReleaseAvailable(release) + return release ? { organization: { ...organization, slug: organization.slug }, site, @@ -246,10 +245,7 @@ export const getPageMetadata = query({ if (!resolved) return null; return { title: resolved.page.title, - descriptionText: - resolved.page.description || - resolved.page.descriptionText || - resolved.page.title, + descriptionText: resolved.page.description || resolved.page.title, canonicalPath: canonicalPagePath(context.release, resolved), updatedAt: resolved.page.updatedAt, }; @@ -364,7 +360,7 @@ export const sitemap = query({ const release = site.liveReleaseId ? await ctx.db.get(site.liveReleaseId) : null; - if (!release || !isReleaseAvailable(release)) return null; + if (!release) return null; const pages = await ctx.db .query("releasePages") .withIndex("by_release", (q) => q.eq("releaseId", release._id)) @@ -417,7 +413,7 @@ export const getPageExport = query({ const access = await resolvePublishedSiteAccess(ctx, site); if (!canRenderPublishedSite(access)) return null; const release = await ctx.db.get(site.liveReleaseId); - if (!release || !isReleaseAvailable(release)) return null; + if (!release) return null; const page = await ctx.db .query("releasePages") .withIndex("by_release_page", (q) => diff --git a/packages/backend/convex/releasePublication.test.ts b/packages/backend/convex/releasePublication.test.ts deleted file mode 100644 index f8b7464f..00000000 --- a/packages/backend/convex/releasePublication.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { internal } from "./_generated/api"; -import { cleanupFailedRelease, recover } from "./releasePublication"; - -type RegisteredFunction = { - _handler: (ctx: unknown, args: unknown) => Promise; -}; - -function invoke(fn: unknown, ctx: unknown, args: unknown): Promise { - return (fn as RegisteredFunction)._handler(ctx, args); -} - -type Row = Record & { _id: string }; - -function makeContext(tables: Record) { - const patches: Array<[string, Record]> = []; - const deleted: string[] = []; - const scheduled: Array<{ - delay: number; - functionReference: unknown; - args: Record; - }> = []; - - return { - patches, - deleted, - scheduled, - db: { - get: async (id: string) => { - for (const rows of Object.values(tables)) { - const row = rows.find((candidate) => candidate._id === id); - if (row) return row; - } - return null; - }, - patch: async (id: string, values: Record) => { - for (const rows of Object.values(tables)) { - const row = rows.find((candidate) => candidate._id === id); - if (row) Object.assign(row, values); - } - patches.push([id, values]); - }, - delete: async (id: string) => { - for (const rows of Object.values(tables)) { - const index = rows.findIndex((candidate) => candidate._id === id); - if (index >= 0) rows.splice(index, 1); - } - deleted.push(id); - }, - query: (table: string) => ({ - withIndex: () => ({ - collect: async () => tables[table] ?? [], - paginate: async () => ({ - page: tables[table] ?? [], - isDone: true, - continueCursor: "", - }), - }), - }), - }, - scheduler: { - runAfter: async ( - delay: number, - functionReference: unknown, - args: Record, - ) => { - scheduled.push({ delay, functionReference, args }); - }, - }, - }; -} - -describe("legacy publication workflow recovery", () => { - test("marks an interrupted building release unavailable", async () => { - const release: Row = { - _id: "release-1", - siteId: "site-1", - publicationStatus: "building", - }; - const state = makeContext({ siteReleases: [release] }); - - await invoke(recover, state, { releaseId: release._id }); - - expect(release).toMatchObject({ - publicationStatus: "failed", - publicationFailure: - "The legacy publication was interrupted during the publishing refactor.", - }); - expect(state.scheduled).toEqual([ - { - delay: 0, - functionReference: internal.releasePublication.cleanupFailedRelease, - args: { releaseId: release._id, phase: "pages" }, - }, - ]); - }); - - test("cleans partial release snapshots in bounded phases", async () => { - const release: Row = { - _id: "release-1", - siteId: "site-1", - publicationStatus: "failed", - }; - const tables = { - siteReleases: [release], - releasePages: [{ _id: "page-snapshot", releaseId: release._id }], - releaseLibraries: [{ _id: "library-snapshot", releaseId: release._id }], - releaseFolders: [{ _id: "folder-snapshot", releaseId: release._id }], - releaseFiles: [{ _id: "file-snapshot", releaseId: release._id }], - searchEntries: [ - { _id: "search-snapshot", scopeId: `release:${release._id}` }, - ], - releaseChanges: [{ _id: "change-snapshot", releaseId: release._id }], - } satisfies Record; - const state = makeContext(tables); - - for (const phase of [ - "pages", - "libraries", - "folders", - "files", - "search", - "changes", - ] as const) { - await invoke(cleanupFailedRelease, state, { - releaseId: release._id, - phase, - }); - } - - expect(tables.releasePages).toHaveLength(0); - expect(tables.releaseLibraries).toHaveLength(0); - expect(tables.releaseFolders).toHaveLength(0); - expect(tables.releaseFiles).toHaveLength(0); - expect(tables.searchEntries).toHaveLength(0); - expect(tables.releaseChanges).toHaveLength(0); - }); - - test("finishes cleanup only for the exact snapshotted draft generation", async () => { - const release: Row = { - _id: "release-1", - siteId: "site-1", - publicationStatus: "clearing", - }; - const site: Row = { - _id: "site-1", - liveReleaseId: release._id, - }; - const state = makeContext({ - siteReleases: [release], - sites: [site], - releaseChanges: [ - { - _id: "snapshot-1", - releaseId: release._id, - sourceDraftChangeId: "change-1", - sourceDraftRevision: 7, - }, - { - _id: "snapshot-2", - releaseId: release._id, - sourceDraftChangeId: "change-2", - sourceDraftRevision: 6, - }, - ], - draftChanges: [ - { _id: "change-1", siteId: site._id, draftRevision: 7 }, - { _id: "change-2", siteId: site._id, draftRevision: 8 }, - { _id: "change-3", siteId: site._id, draftRevision: 7 }, - ], - }); - - await invoke(recover, state, { releaseId: release._id }); - - expect(release.publicationStatus).toBe("complete"); - expect(state.deleted).toEqual(["change-1"]); - expect(state.scheduled).toEqual([ - { - delay: 0, - functionReference: internal.publication.projectLiveSearch, - args: { siteId: site._id, expectedLiveReleaseId: release._id }, - }, - ]); - }); -}); diff --git a/packages/backend/convex/releasePublication.ts b/packages/backend/convex/releasePublication.ts deleted file mode 100644 index 5328a864..00000000 --- a/packages/backend/convex/releasePublication.ts +++ /dev/null @@ -1,216 +0,0 @@ -import { v } from "convex/values"; -import { internal } from "./_generated/api"; -import type { Id } from "./_generated/dataModel"; -import { internalMutation } from "./_generated/server"; -import type { MutationCtx } from "./_generated/server"; -import { workflows } from "./workflows"; - -const LEGACY_CLEANUP_BATCH_SIZE = 50; - -const legacyCleanupPhase = v.union( - v.literal("pages"), - v.literal("libraries"), - v.literal("folders"), - v.literal("files"), - v.literal("search"), - v.literal("changes"), -); - -type LegacyCleanupPhase = - | "pages" - | "libraries" - | "folders" - | "files" - | "search" - | "changes"; - -const nextLegacyCleanupPhase: Record< - LegacyCleanupPhase, - LegacyCleanupPhase | undefined -> = { - pages: "libraries", - libraries: "folders", - folders: "files", - files: "search", - search: "changes", - changes: undefined, -}; - -async function scheduleLegacyCleanup( - ctx: Pick, - args: { - releaseId: Id<"siteReleases">; - phase: LegacyCleanupPhase; - cursor?: string; - }, -) { - await ctx.scheduler.runAfter( - 0, - internal.releasePublication.cleanupFailedRelease, - args, - ); -} - -/** - * Compatibility entry point for workflows created before publication became - * atomic. New releases never use this workflow. Keeping the function path - * available lets an already queued workflow finish safely after deployment. - */ -export const recover = internalMutation({ - args: { releaseId: v.id("siteReleases") }, - returns: v.null(), - handler: async (ctx, { releaseId }) => { - const release = await ctx.db.get(releaseId); - if (!release || release.publicationStatus === undefined) return null; - - const now = Date.now(); - if (release.publicationStatus === "clearing") { - const snapshots = await ctx.db - .query("releaseChanges") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .collect(); - const snapshottedChanges = new Map( - snapshots - .filter( - (snapshot) => - snapshot.sourceDraftChangeId !== undefined && - snapshot.sourceDraftRevision !== undefined, - ) - .map((snapshot) => [ - snapshot.sourceDraftChangeId!, - snapshot.sourceDraftRevision!, - ]), - ); - const currentChanges = await ctx.db - .query("draftChanges") - .withIndex("by_site", (q) => q.eq("siteId", release.siteId)) - .collect(); - for (const change of currentChanges) { - if (snapshottedChanges.get(change._id) === change.draftRevision) { - await ctx.db.delete(change._id); - } - } - await ctx.db.patch(releaseId, { - publicationStatus: "complete", - publicationFailure: undefined, - publicationUpdatedAt: now, - }); - const site = await ctx.db.get(release.siteId); - if (site?.liveReleaseId === releaseId) { - await ctx.scheduler.runAfter( - 0, - internal.publication.projectLiveSearch, - { - siteId: site._id, - expectedLiveReleaseId: releaseId, - expectedLiveSearchProjectionGeneration: - site.liveSearchProjectionGeneration, - }, - ); - } - return null; - } - - if (release.publicationStatus === "building") { - await ctx.db.patch(releaseId, { - publicationStatus: "failed", - publicationFailure: - "The legacy publication was interrupted during the publishing refactor.", - publicationUpdatedAt: now, - }); - await scheduleLegacyCleanup(ctx, { - releaseId, - phase: "pages", - }); - } - return null; - }, -}); - -/** - * Removes partial snapshots left by an interrupted legacy build without - * placing a large cleanup in one transaction. The failed release row remains - * as an audit record and is never made readable by the publication surface. - */ -async function readLegacyCleanupPage( - ctx: MutationCtx, - releaseId: Id<"siteReleases">, - phase: LegacyCleanupPhase, - cursor: string | undefined, -) { - const paginationOpts = { - cursor: cursor ?? null, - numItems: LEGACY_CLEANUP_BATCH_SIZE, - }; - switch (phase) { - case "pages": - return ctx.db - .query("releasePages") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .paginate(paginationOpts); - case "libraries": - return ctx.db - .query("releaseLibraries") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .paginate(paginationOpts); - case "folders": - return ctx.db - .query("releaseFolders") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .paginate(paginationOpts); - case "files": - return ctx.db - .query("releaseFiles") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .paginate(paginationOpts); - case "search": - return ctx.db - .query("searchEntries") - .withIndex("by_scope", (q) => q.eq("scopeId", `release:${releaseId}`)) - .paginate(paginationOpts); - case "changes": - return ctx.db - .query("releaseChanges") - .withIndex("by_release", (q) => q.eq("releaseId", releaseId)) - .paginate(paginationOpts); - } -} - -export const cleanupFailedRelease = internalMutation({ - args: { - releaseId: v.id("siteReleases"), - phase: legacyCleanupPhase, - cursor: v.optional(v.string()), - }, - returns: v.null(), - handler: async (ctx, { releaseId, phase, cursor }) => { - const release = await ctx.db.get(releaseId); - if (release?.publicationStatus !== "failed") return null; - - const page = await readLegacyCleanupPage(ctx, releaseId, phase, cursor); - - for (const row of page.page) await ctx.db.delete(row._id); - if (!page.isDone) { - await scheduleLegacyCleanup(ctx, { - releaseId, - phase, - cursor: page.continueCursor, - }); - return null; - } - - const nextPhase = nextLegacyCleanupPhase[phase]; - if (nextPhase) { - await scheduleLegacyCleanup(ctx, { releaseId, phase: nextPhase }); - } - return null; - }, -}); - -export const run = workflows - .define({ args: { releaseId: v.id("siteReleases") } }) - .handler(async (step, { releaseId }): Promise => { - await step.runMutation(internal.releasePublication.recover, { - releaseId, - }); - }); diff --git a/packages/backend/convex/releases.ts b/packages/backend/convex/releases.ts index a9024f85..df2580e9 100644 --- a/packages/backend/convex/releases.ts +++ b/packages/backend/convex/releases.ts @@ -1,14 +1,9 @@ import { ConvexError, v } from "convex/values"; import type { GenericMutationCtx } from "convex/server"; -import { getStatus, type WorkflowId } from "@convex-dev/workflow"; -import { components, internal } from "./_generated/api"; +import { internal } from "./_generated/api"; import type { DataModel, Id } from "./_generated/dataModel"; import { mutation, query } from "./_generated/server"; -import { - extractionBlocksPublication, - isPublicationInFlight, - isReleaseAvailable, -} from "./model/releaseState"; +import { extractionBlocksPublication } from "./model/releaseState"; import { buildDraftSummary } from "./model/draftSummary"; import { promoteRelease } from "./model/releaseOperations"; import { buildHistoricalReleaseContent } from "./model/releaseChangeDetails"; @@ -110,58 +105,6 @@ export const getDraftChanges = query({ }, }); -/** - * Backward-compatible read for clients that still poll the retired workflow. - * Atomic releases have no workflow status and are complete at creation time. - */ -export const getPublicationStatus = query({ - args: { releaseId: v.id("siteReleases") }, - returns: v.union( - v.null(), - v.object({ - status: v.union( - v.literal("building"), - v.literal("clearing"), - v.literal("complete"), - v.literal("failed"), - ), - failure: v.optional(v.string()), - }), - ), - handler: async (ctx, { releaseId }) => { - const release = await ctx.db.get(releaseId); - if (!release || !(await requireSiteForMember(ctx, release.siteId))) { - return null; - } - if ( - isPublicationInFlight(release.publicationStatus) && - !release.publicationWorkflowId - ) { - return { - status: "failed" as const, - failure: "Publication workflow state is missing", - }; - } - if ( - isPublicationInFlight(release.publicationStatus) && - release.publicationWorkflowId - ) { - const workflowStatus = await getStatus( - ctx, - components.workflow, - release.publicationWorkflowId as WorkflowId, - ); - if (workflowStatus.type === "failed") { - return { status: "failed" as const, failure: workflowStatus.error }; - } - } - return { - status: release.publicationStatus ?? ("complete" as const), - failure: release.publicationFailure, - }; - }, -}); - export const list = query({ args: { siteId: v.id("sites") }, returns: v.array(releaseSummaryValidator), @@ -173,9 +116,9 @@ export const list = query({ .withIndex("by_site_number", (q) => q.eq("siteId", siteId)) .order("desc") .collect(); - return releases - .filter(isReleaseAvailable) - .map((release) => releaseSummary(release, site.liveReleaseId)); + return releases.map((release) => + releaseSummary(release, site.liveReleaseId), + ); }, }); @@ -210,7 +153,7 @@ export const get = query({ ), handler: async (ctx, { releaseId }) => { const release = await ctx.db.get(releaseId); - if (!release || !isReleaseAvailable(release)) return null; + if (!release) return null; const site = await requireSiteForMember(ctx, release.siteId); if (!site) return null; const changes = await ctx.db @@ -272,11 +215,7 @@ export const publish = mutation({ const matchingRelease = site.draftBaseReleaseId ? await ctx.db.get(site.draftBaseReleaseId) : null; - if ( - matchingRelease && - (matchingRelease.siteId !== siteId || - !isReleaseAvailable(matchingRelease)) - ) { + if (matchingRelease && matchingRelease.siteId !== siteId) { throw new ConvexError("The draft base version is unavailable"); } const pendingChange = await ctx.db @@ -452,9 +391,6 @@ export const makeLive = mutation({ site.organizationId, { resource: "publication", action: "publish" }, ); - if (!isReleaseAvailable(release)) { - throw new ConvexError("Release publication is not complete"); - } if (site.activeDraftRestoreId) { throw new ConvexError( "A historical version is currently being restored. Try again when it finishes.", diff --git a/packages/backend/convex/schema.ts b/packages/backend/convex/schema.ts index f8f7d7d1..a9868738 100644 --- a/packages/backend/convex/schema.ts +++ b/packages/backend/convex/schema.ts @@ -93,7 +93,7 @@ export default defineSchema({ * Publication, search projection, and metadata reads consume this * instead of re-parsing the payload. */ - searchText: v.optional(v.string()), + searchText: v.string(), libraryIds: v.array(v.id("documentLibraries")), fileIds: v.array(v.id("files")), pageIds: v.array(v.id("pages")), @@ -114,39 +114,6 @@ export default defineSchema({ .index("by_page", ["pageId"]) .index("by_revision", ["revisionId"]), - /** - * Progress for the publication manifest backfill. The migration is kept - * separate from the legacy publication workflow so it can be dry-run and - * resumed without changing publication behavior. - */ - publicationMigrationRuns: defineTable({ - migrationKey: v.string(), - runId: v.string(), - mode: v.union(v.literal("dryRun"), v.literal("apply")), - phase: v.union( - v.literal("revisions"), - v.literal("pages"), - v.literal("files"), - v.literal("search"), - v.literal("sites"), - ), - cursor: v.optional(v.string()), - status: v.union( - v.literal("running"), - v.literal("completed"), - v.literal("failed"), - ), - scannedCount: v.number(), - migratedCount: v.number(), - skippedCount: v.number(), - scheduledCount: v.optional(v.number()), - errorCount: v.number(), - startedAt: v.number(), - updatedAt: v.number(), - completedAt: v.optional(v.number()), - failureSummary: v.optional(v.string()), - }).index("by_migration_run", ["migrationKey", "runId"]), - draftChanges: defineTable({ siteId: v.id("sites"), entityType: v.union( @@ -404,21 +371,6 @@ export default defineSchema({ createdAt: v.number(), pageCount: v.number(), changeCount: v.number(), - /** - * Legacy publication workflow fields remain optional so existing rows can - * still be read while new releases use atomic activation. - */ - publicationStatus: v.optional( - v.union( - v.literal("building"), - v.literal("clearing"), - v.literal("complete"), - v.literal("failed"), - ), - ), - publicationFailure: v.optional(v.string()), - publicationWorkflowId: v.optional(v.string()), - publicationUpdatedAt: v.optional(v.number()), }) .index("by_site", ["siteId"]) .index("by_site_number", ["siteId", "number"]), @@ -460,9 +412,7 @@ export default defineSchema({ * revision's captured search text. The live projection owns its searchable * copy; the release row keeps only this metadata snippet. */ - description: v.optional(v.string()), - /** Compatibility with releases created before the description rename. */ - descriptionText: v.optional(v.string()), + description: v.string(), updatedAt: v.number(), }) .index("by_content_revision", ["contentRevisionId"]) diff --git a/packages/backend/convex/search.ts b/packages/backend/convex/search.ts index 00d15158..9781fe04 100644 --- a/packages/backend/convex/search.ts +++ b/packages/backend/convex/search.ts @@ -5,7 +5,6 @@ import type { DataModel, Doc, Id } from "./_generated/dataModel"; import { internalMutation, query } from "./_generated/server"; import { readContentRevisionSearchText } from "./model/contentObjects"; import { assertDraftReadable } from "./model/draft"; -import { isReleaseAvailable } from "./model/releaseState"; import { isOrganizationMember } from "./permissions"; import { canRenderPublishedSite, resolvePublishedSiteAccess } from "./sharing"; @@ -35,11 +34,6 @@ export function liveSearchScope(siteId: Id<"sites">): string { return `live:${siteId}`; } -/** Compatibility scope used by releases created before live projection. */ -export function releaseSearchScope(releaseId: Id<"siteReleases">): string { - return `release:${releaseId}`; -} - export function isPublishedSearchEntryForRelease( entry: Pick, "releaseId">, releaseId: Id<"siteReleases">, @@ -348,7 +342,7 @@ export const run = query({ const access = await resolvePublishedSiteAccess(ctx, site); if (!canRenderPublishedSite(access) || !site.liveReleaseId) return []; const release = await ctx.db.get(site.liveReleaseId); - if (!release || !isReleaseAvailable(release)) return []; + if (!release) return []; releaseId = site.liveReleaseId; scopeId = liveSearchScope(siteId); } @@ -359,22 +353,11 @@ export const run = query({ searchTerm, normalizeSearchLimit(limit), ); - let currentMatches = releaseId + const currentMatches = releaseId ? matches.filter(({ doc }) => isPublishedSearchEntryForRelease(doc, releaseId), ) : matches; - if (releaseId && currentMatches.length === 0) { - const legacyMatches = await searchScope( - ctx, - releaseSearchScope(releaseId), - searchTerm, - normalizeSearchLimit(limit), - ); - if (legacyMatches.length > 0) { - currentMatches = legacyMatches; - } - } const hydrated = await Promise.all( currentMatches.map(({ doc, match }) => releaseId diff --git a/packages/backend/convex/siteDomains.ts b/packages/backend/convex/siteDomains.ts index 158f09e1..f5171b87 100644 --- a/packages/backend/convex/siteDomains.ts +++ b/packages/backend/convex/siteDomains.ts @@ -1,7 +1,6 @@ import { v } from "convex/values"; import { mutation, query } from "./_generated/server"; import { getAuthOrganizationById } from "./authComponent/model"; -import { isReleaseAvailable } from "./model/releaseState"; import { requireOrganizationPermission } from "./permissions"; const domainStatus = v.union( @@ -22,7 +21,7 @@ export const resolve = query({ const site = await ctx.db.get(mapping.siteId); if (!site?.liveReleaseId) return null; const release = await ctx.db.get(site.liveReleaseId); - if (!release || !isReleaseAvailable(release)) return null; + if (!release) return null; const organization = await getAuthOrganizationById( ctx, site.organizationId,