diff --git a/apps/api/src/decofile/repo-content-storage.ts b/apps/api/src/decofile/repo-content-storage.ts new file mode 100644 index 0000000000..01e24c05cd --- /dev/null +++ b/apps/api/src/decofile/repo-content-storage.ts @@ -0,0 +1,191 @@ +/** + * The read side of the content protocol over a project's Git repository: the + * file list, file bodies, schema and secrets key of one branch. The hosted v8 + * editor (`hosted/draft-content-storage.ts`) reads the default branch through + * it and layers the CDN draft on top; nothing here writes. + * + * Versions are git blob shas and the revision is the branch head sha. A + * missing branch reads as the default branch (reported in `resolvedRef`). + */ + +import { + type ContentStorage, + type StorageSnapshot, + type StoredFileBody, + type StoredSchema, + StorageNotFoundError, + StorageUnavailableError, +} from "@decocms/blocks/protocol"; +import { + type RepoContentClient, + repoRateLimitRetryAfterMs, + requireBranchHead, +} from "@/git-providers"; +import { + blockEntriesInTree, + blocksDirPath, + resolveBlockContents, +} from "./read-decofile"; + +/** What the hosted draft storage reads from the repository. */ +export type RepoContentReader = Pick< + ContentStorage, + "snapshot" | "readFiles" | "readSchema" | "readSecretsPublicKey" +>; + +interface RepoContentStorageOptions { + client: RepoContentClient; + /** The app root inside the repository; `null` at the repository root. */ + packagePath: string | null; + /** The one branch this storage reads. */ + branch: string; +} + +/** + * Whether committed schema text is a Blocks v8 one: only a top-level + * `"blocksMajor": 8` (written by `deco schema`) counts. Missing, any other + * value or unparseable text is a v7 site, which this protocol never serves. + * Mirrors the web's `isV8Schema`. + */ +function isV8SchemaText(text: string): boolean { + try { + const schema: unknown = JSON.parse(text); + return ( + typeof schema === "object" && + schema !== null && + !Array.isArray(schema) && + (schema as { blocksMajor?: unknown }).blocksMajor === 8 + ); + } catch { + return false; + } +} + +function decoPath(packagePath: string | null, file: string): string { + return packagePath ? `${packagePath}/.deco/${file}` : `.deco/${file}`; +} + +/** Turns provider failures into the storage errors the core understands. */ +async function guarded(work: () => Promise): Promise { + try { + return await work(); + } catch (error) { + const retryAfterMs = repoRateLimitRetryAfterMs(error); + if (retryAfterMs !== undefined) { + throw new StorageUnavailableError( + "the Git provider is rate-limiting this repository", + retryAfterMs ?? undefined, + ); + } + throw error; + } +} + +export function createRepoContentReader( + options: RepoContentStorageOptions, +): RepoContentReader { + const { client, packagePath, branch } = options; + const blocksPrefix = `${blocksDirPath(packagePath)}/`; + + /** The bound branch's head, or the default branch's while it doesn't exist. */ + const readHead = async (): Promise<{ sha: string; ref: string }> => { + const head = await client.getBranch(branch); + if (head) return { sha: head.sha, ref: branch }; + const defaultBranch = await client.getDefaultBranch(); + return { + sha: await requireBranchHead(client, defaultBranch), + ref: defaultBranch, + }; + }; + + /** `schema.gen.json`, else `meta.gen.json`, at one commit. */ + const schemaEntryAt = async (sha: string) => { + const paths = [ + decoPath(packagePath, "schema.gen.json"), + decoPath(packagePath, "meta.gen.json"), + ]; + const entries = await client.getEntriesAtPaths(sha, paths); + for (const path of paths) { + const entry = entries.get(path); + if (entry?.type === "blob") return entry; + } + return null; + }; + + /** The schema the bound branch sees, falling back to the default branch's. */ + const resolveSchema = async (head: { sha: string; ref: string }) => { + const entry = await schemaEntryAt(head.sha); + if (entry) return { entry, ref: head.ref }; + const defaultBranch = await client.getDefaultBranch(); + if (defaultBranch === head.ref) return null; + const fallback = await schemaEntryAt( + await requireBranchHead(client, defaultBranch), + ); + return fallback ? { entry: fallback, ref: defaultBranch } : null; + }; + + const readBlobText = async (stem: string, sha: string): Promise => { + const [file] = await resolveBlockContents(client, [{ stem, sha }]); + return file!.content; + }; + + return { + snapshot: () => + guarded(async (): Promise => { + const head = await readHead(); + const tree = await client.listDecofileEntries(head.sha, packagePath); + if (tree.length === 0 && (await schemaEntryAt(head.sha)) === null) { + throw new StorageNotFoundError( + `no .deco folder in ${packagePath ?? "the repository root"}`, + ); + } + return { + revision: head.sha, + resolvedRef: head.ref, + files: blockEntriesInTree(tree, packagePath).map((entry) => ({ + file: entry.path.slice(blocksPrefix.length), + version: entry.sha, + size: entry.size, + })), + }; + }), + + readFiles: (snapshot, files) => + guarded(async () => { + const wanted = new Set(files); + const sources = snapshot.files.filter((f) => wanted.has(f.file)); + // A snapshot is a commit, so the bytes always match its versions. + const contents = await resolveBlockContents( + client, + sources.map((f) => ({ stem: f.file, sha: f.version })), + ); + const out: Record = {}; + sources.forEach((f, i) => { + out[f.file] = { text: contents[i]!.content, version: f.version }; + }); + return out; + }), + + readSchema: () => + guarded(async (): Promise => { + const schema = await resolveSchema(await readHead()); + if (!schema) return null; + const text = await readBlobText(schema.entry.path, schema.entry.sha); + // A v7 site reads as schemaless, so the editor stays on the classic one. + if (!isV8SchemaText(text)) return null; + return { version: schema.entry.sha, text, resolvedRef: schema.ref }; + }), + + readSecretsPublicKey: () => + guarded(async () => { + const head = await readHead(); + const path = decoPath(packagePath, "secrets.pub"); + const entry = (await client.getEntriesAtPaths(head.sha, [path])).get( + path, + ); + return entry?.type === "blob" + ? readBlobText(entry.path, entry.sha) + : null; + }), + }; +} diff --git a/apps/api/src/hosted/draft-content-storage.test.ts b/apps/api/src/hosted/draft-content-storage.test.ts new file mode 100644 index 0000000000..122f58d45a --- /dev/null +++ b/apps/api/src/hosted/draft-content-storage.test.ts @@ -0,0 +1,210 @@ +import { beforeAll, describe, expect, it } from "bun:test"; +import { serializeBlock } from "@decocms/blocks/protocol"; +import { deliveryKeys } from "./delivery-store"; +import { + applyAttemptToDraft, + createDraftContentStorage, +} from "./draft-content-storage"; +import { createDraftStore, type HostedDraftRef } from "./draft-store"; +import { + fakeRepo, + memoryDeliveryStore, + memoryKv, + SCHEMA_TEXT, +} from "./hosted-test-helpers"; + +beforeAll(() => { + process.env.FAST_PREVIEW_CACHE_DIR = ""; +}); + +const REF: HostedDraftRef = { + organizationId: "org", + virtualMcpId: "vmcp", + branch: "thread-1", + site: "acme", +}; + +function setup() { + const repo = fakeRepo({ + ".deco/schema.gen.json": SCHEMA_TEXT, + ".deco/blocks/Home.json": serializeBlock({ path: "/", title: "main" }), + ".deco/blocks/Footer.json": serializeBlock({ links: [] }), + }); + const delivery = memoryDeliveryStore(); + const drafts = createDraftStore({ kv: memoryKv(), store: delivery.store }); + const storage = createDraftContentStorage({ + client: repo.client, + packagePath: null, + mainBranch: "main", + drafts, + ref: REF, + }); + return { repo, delivery, drafts, storage }; +} + +describe("applyAttemptToDraft", () => { + const main = new Set(["Home", "Footer"]); + + it("puts set the block and lift a pending delete", () => { + const next = applyAttemptToDraft( + { set: {}, delete: ["Home"] }, + { put: { "Home.json": '{"a":1}' }, delete: [] }, + main, + ); + expect(next).toEqual({ set: { Home: { a: 1 } }, delete: [] }); + }); + + it("deleting a main block moves it to delete; a draft-only one just goes", () => { + const next = applyAttemptToDraft( + { set: { Home: { a: 1 }, New: { b: 2 } }, delete: [] }, + { put: {}, delete: ["Home.json", "New.json"] }, + main, + ); + expect(next).toEqual({ set: {}, delete: ["Home"] }); + }); + + it("ignores the shadow-spelling deletes of a written name", () => { + const next = applyAttemptToDraft( + { set: {}, delete: [] }, + { put: { "a%20b.json": '{"x":1}' }, delete: ["a%2520b.json"] }, + new Set(["a b"]), + ); + expect(next).toEqual({ set: { "a b": { x: 1 } }, delete: [] }); + }); +}); + +describe("createDraftContentStorage", () => { + it("reads main with no draft, at revision ~none", async () => { + const { repo, storage } = setup(); + const snap = await storage.snapshot(); + expect(snap.revision).toBe(`${repo.head()}~none`); + expect(snap.files.map((f) => f.file).sort()).toEqual([ + "Footer.json", + "Home.json", + ]); + }); + + it("saves into the CDN draft, never into git, and reads it back layered", async () => { + const { repo, delivery, storage } = setup(); + const before = repo.head(); + const snap = await storage.snapshot(); + const result = await storage.commit({ + base: snap, + put: { "Home.json": serializeBlock({ path: "/", title: "draft" }) }, + delete: ["Footer.json"], + expected: {}, + }); + expect(result.status).toBe("committed"); + expect(repo.head()).toBe(before); + const [key] = await delivery.store.listKeys("sites/acme/drafts/"); + const saved = delivery.objects.get(key!)!; + expect(JSON.parse(saved.text)).toEqual({ + set: { Home: { path: "/", title: "draft" } }, + delete: ["Footer"], + }); + expect(saved.cacheControl).toBe("no-cache, max-age=0, must-revalidate"); + + const next = await storage.snapshot(); + expect(next.revision).toBe(`${before}~${saved.etag.replaceAll('"', "")}`); + expect(result).toMatchObject({ revision: next.revision }); + expect(next.files.map((f) => f.file)).toEqual(["Home.json"]); + const read = await storage.readFiles(next, ["Home.json"]); + expect(JSON.parse(read["Home.json"]!.text)).toEqual({ + path: "/", + title: "draft", + }); + }); + + it("the last writer wins: a save over another's base still lands", async () => { + const { storage } = setup(); + const stale = await storage.snapshot(); + await storage.commit({ + base: stale, + put: { "A.json": "{}" }, + delete: [], + expected: {}, + }); + const result = await storage.commit({ + base: stale, + put: { "B.json": "{}" }, + delete: [], + expected: { "B.json": null }, + }); + expect(result.status).toBe("committed"); + const files = (await storage.snapshot()).files.map((f) => f.file).sort(); + expect(files).toEqual(["A.json", "B.json", "Footer.json", "Home.json"]); + }); + + it("is stale when main's schema changed under the save", async () => { + const { storage } = setup(); + const result = await storage.commit({ + base: await storage.snapshot(), + put: { "A.json": "{}" }, + delete: [], + expected: {}, + expectedSchemaVersion: "another", + }); + expect(result).toEqual({ status: "stale" }); + }); + + it("refuses to write a site whose main isn't v8", async () => { + const repo = fakeRepo({ ".deco/blocks/Home.json": "{}" }); + const delivery = memoryDeliveryStore(); + const storage = createDraftContentStorage({ + client: repo.client, + packagePath: null, + mainBranch: "main", + drafts: createDraftStore({ kv: memoryKv(), store: delivery.store }), + ref: REF, + }); + await expect( + storage.commit({ + base: await storage.snapshot(), + put: { "A.json": "{}" }, + delete: [], + expected: {}, + }), + ).rejects.toThrow(); + expect(delivery.objects.size).toBe(0); + }); +}); + +describe("createDraftStore", () => { + it("keeps one slug per (project, branch) until the draft is removed", async () => { + const delivery = memoryDeliveryStore(); + const drafts = createDraftStore({ kv: memoryKv(), store: delivery.store }); + expect(await drafts.load(REF)).toBeNull(); + const first = await drafts.update(REF, () => ({ + set: { A: {} }, + delete: [], + })); + const second = await drafts.update(REF, (b) => ({ ...b, delete: ["B"] })); + expect(second.slug).toBe(first.slug); + expect((await drafts.load(REF))?.body).toEqual({ + set: { A: {} }, + delete: ["B"], + }); + const other = await drafts.update({ ...REF, branch: "thread-2" }, (b) => b); + expect(other.slug).not.toBe(first.slug); + await drafts.remove(REF); + expect(await drafts.load(REF)).toBeNull(); + expect(delivery.objects.has(deliveryKeys.draft("acme", first.slug))).toBe( + false, + ); + }); + + it("serializes concurrent saves of one draft in this process", async () => { + const delivery = memoryDeliveryStore(); + const drafts = createDraftStore({ kv: memoryKv(), store: delivery.store }); + await Promise.all( + ["A", "B", "C"].map((name) => + drafts.update(REF, (b) => ({ ...b, set: { ...b.set, [name]: {} } })), + ), + ); + expect(Object.keys((await drafts.load(REF))!.body.set).sort()).toEqual([ + "A", + "B", + "C", + ]); + }); +}); diff --git a/apps/api/src/hosted/draft-content-storage.ts b/apps/api/src/hosted/draft-content-storage.ts new file mode 100644 index 0000000000..283d247f07 --- /dev/null +++ b/apps/api/src/hosted/draft-content-storage.ts @@ -0,0 +1,204 @@ +/** + * The site editor's content-protocol storage for a hosted v8 project on + * GitHub: the default branch's saved blocks with the project's CDN draft + * layered on top, the way the site's SDK layers it for a preview. + * + * - A draft `set` name replaces every spelling of that name with one file, + * `blockFileName(name)`; a `delete` name hides every spelling. + * - The revision is `
~` (`~none` without a draft), so + * a save or a new main commit is a new revision. + * - A commit edits the draft, never git: put X sets `set[X]` and drops X from + * `delete`; deleting X that exists on main moves it to `delete`; deleting a + * draft-only X just drops it from `set`. The last writer wins (no stale + * check against the draft); the schema is still checked against main's. + */ + +import { + blockFileName, + blockNameFromFile, + type CommitResult, + type ContentStorage, + type StorageDescription, + type StorageFile, + type StorageSnapshot, + type StoredFileBody, + fullyDecodeFileName, + serializeBlock, + spellingKey, + unsupported, +} from "@decocms/blocks/protocol"; +import type { RepoContentClient } from "@/git-providers"; +import { gitBlobSha } from "@/decofile/read-decofile"; +import { createRepoContentReader } from "@/decofile/repo-content-storage"; +import { bareEtag } from "./delivery-store"; +import type { DraftBody, DraftStore, HostedDraftRef } from "./draft-store"; + +/** The spelling group (fully decoded name) a saved-block file belongs to. */ +function groupOf(file: string): string { + return fullyDecodeFileName(file).name; +} + +function draftRevision(mainSha: string, etag: string | null): string { + return `${mainSha}~${bareEtag(etag) ?? "none"}`; +} + +/** Main's files with the draft layered on; returns the draft files' texts too. */ +function layerDraftFiles( + mainFiles: StorageFile[], + draft: DraftBody, +): { files: StorageFile[]; texts: Map } { + const touched = new Set( + [...Object.keys(draft.set), ...draft.delete].map(spellingKey), + ); + const files = mainFiles.filter((f) => !touched.has(groupOf(f.file))); + const texts = new Map(); + for (const [name, entry] of Object.entries(draft.set)) { + const file = blockFileName(name); + const text = serializeBlock(entry); + texts.set(file, text); + files.push({ + file, + version: gitBlobSha(text), + size: Buffer.byteLength(text), + }); + } + return { files, texts }; +} + +/** + * The draft after one commit attempt (see the module note). `mainGroups` are + * the spelling groups that exist on main. + */ +export function applyAttemptToDraft( + draft: DraftBody, + attempt: { put: Record; delete: string[] }, + mainGroups: ReadonlySet, +): DraftBody { + const set = { ...draft.set }; + const deleted = new Set(draft.delete); + const dropGroup = (group: string) => { + for (const name of Object.keys(set)) { + if (spellingKey(name) === group) delete set[name]; + } + for (const name of deleted) { + if (spellingKey(name) === group) deleted.delete(name); + } + }; + const putGroups = new Set(); + for (const [file, content] of Object.entries(attempt.put)) { + const group = groupOf(file); + putGroups.add(group); + dropGroup(group); + set[blockNameFromFile(file)] = JSON.parse(content) as unknown; + } + for (const file of attempt.delete) { + const group = groupOf(file); + // The core deletes a written name's other spellings in the same attempt. + if (putGroups.has(group)) continue; + dropGroup(group); + if (mainGroups.has(group)) deleted.add(blockNameFromFile(file)); + } + return { + set: Object.fromEntries(Object.entries(set)), + delete: [...deleted].sort(), + }; +} + +export function createDraftContentStorage(options: { + client: RepoContentClient; + packagePath: string | null; + /** The default branch: what the draft is layered on and published to. */ + mainBranch: string; + drafts: DraftStore; + ref: HostedDraftRef; +}): ContentStorage { + const { drafts, ref } = options; + const main = createRepoContentReader({ + client: options.client, + packagePath: options.packagePath, + branch: options.mainBranch, + }); + /** Draft file texts of the snapshots this storage handed out. */ + const draftTexts = new Map>(); + + const snapshot = async (): Promise => { + const [base, draft] = await Promise.all([ + main.snapshot(), + drafts.load(ref), + ]); + const { files, texts } = layerDraftFiles( + base.files, + draft?.body ?? { set: {}, delete: [] }, + ); + const revision = draftRevision(base.revision, draft?.etag ?? null); + draftTexts.set(revision, texts); + return { revision, resolvedRef: base.resolvedRef, files }; + }; + + return { + describe(): StorageDescription { + const description = { + kind: "git" as const, + root: options.packagePath ?? ".", + readOnly: false, + // Uploads go to Studio's own file storage, never into the repository. + assets: null, + }; + return description; + }, + + snapshot, + + readFiles: async (snap, files) => { + const texts = draftTexts.get(snap.revision) ?? new Map(); + const out: Record = {}; + const fromMain: string[] = []; + for (const file of files) { + const text = texts.get(file); + if (text === undefined) fromMain.push(file); + else out[file] = { text, version: gitBlobSha(text) }; + } + if (fromMain.length > 0) { + Object.assign(out, await main.readFiles(snap, fromMain)); + } + return out; + }, + + readSchema: () => main.readSchema(), + readSecretsPublicKey: () => main.readSecretsPublicKey(), + + commit: async (attempt): Promise => { + // Never write a v7 site through the protocol, even by a direct call. + const schema = await main.readSchema(); + if (!schema) throw unsupported("not a Blocks v8 site"); + if ( + attempt.expectedSchemaVersion !== undefined && + schema.version !== attempt.expectedSchemaVersion + ) { + return { status: "stale" }; + } + const base = await main.snapshot(); + const mainGroups = new Set(base.files.map((f) => groupOf(f.file))); + const versions: Record = {}; + for (const [file, content] of Object.entries(attempt.put)) { + versions[file] = gitBlobSha(serializeBlock(JSON.parse(content))); + } + if (Object.keys(attempt.put).length + attempt.delete.length === 0) { + const current = await drafts.load(ref); + return { + status: "committed", + revision: draftRevision(base.revision, current?.etag ?? null), + versions, + }; + } + const saved = await drafts.update(ref, (body) => + applyAttemptToDraft(body, attempt, mainGroups), + ); + return { + status: "committed", + revision: draftRevision(base.revision, saved.etag), + versions, + }; + }, + }; +} diff --git a/apps/api/src/hosted/draft-store.ts b/apps/api/src/hosted/draft-store.ts new file mode 100644 index 0000000000..b4cd8176c3 --- /dev/null +++ b/apps/api/src/hosted/draft-store.ts @@ -0,0 +1,158 @@ +/** + * A v8 project's editor draft, kept on the delivery bucket at + * `sites//drafts/.json` as `{ set, delete }`: the changed blocks + * (whole), and the names the draft deletes. Saving is a whole-object PUT with + * no git commit; the last writer wins. Publish commits it to main and deletes + * it, so the next edit starts a new slug. + * + * The slug is random (holding the URL is the permission to read the draft). + */ + +import type { KVStorage } from "@/storage/kv"; +import { + CACHE_DRAFT, + type DeliveryStore, + deliveryKeys, + newDraftSlug, +} from "./delivery-store"; + +export interface DraftBody { + set: Record; + delete: string[]; +} + +export interface HostedDraftRef { + organizationId: string; + virtualMcpId: string; + branch: string; + site: string; +} + +export interface LoadedDraft { + slug: string; + body: DraftBody; + etag: string | null; +} + +function emptyDraft(): DraftBody { + return { set: {}, delete: [] }; +} + +export function isEmptyDraft(body: DraftBody): boolean { + return Object.keys(body.set).length === 0 && body.delete.length === 0; +} + +/** Parses a stored draft; throws on anything but `{ set, delete }`. */ +function parseDraftBody(value: unknown): DraftBody { + const v = value as { set?: unknown; delete?: unknown } | null; + if ( + typeof v !== "object" || + v === null || + typeof v.set !== "object" || + v.set === null || + Array.isArray(v.set) || + !Array.isArray(v.delete) || + !v.delete.every((name) => typeof name === "string") + ) { + throw new Error("stored draft isn't { set, delete }"); + } + // fromEntries: an entry named "__proto__" stays an entry. + return { + set: Object.fromEntries(Object.entries(v.set as Record)), + delete: [...(v.delete as string[])], + }; +} + +// OPEN: O-S1 — the draft slug lives in the org KV under (project, branch), +// where the v8 draft branch name used to be implied; no migration needed. +function slugKey(ref: HostedDraftRef): string { + return `v8-draft:${ref.virtualMcpId}:${ref.branch}`; +} + +const locks = new Map>(); + +/** Serializes this process's read-modify-writes of one draft. */ +function withDraftLock(key: string, fn: () => Promise): Promise { + const prior = locks.get(key) ?? Promise.resolve(); + const next = prior.catch(() => {}).then(fn); + const settled = next.catch(() => {}); + locks.set(key, settled); + void settled.finally(() => { + if (locks.get(key) === settled) locks.delete(key); + }); + return next; +} + +export interface DraftStore { + /** The draft, or null when this (project, branch) has none. */ + load(ref: HostedDraftRef): Promise; + /** + * Applies `change` to the current draft (empty when none) and PUTs it, + * creating the slug on the first write. + */ + update( + ref: HostedDraftRef, + change: (body: DraftBody) => DraftBody, + ): Promise; + /** + * Deletes the draft object and forgets its slug. With `expectedEtag`, it + * deletes only while the draft is still that save: a save that landed + * since (e.g. during a publish) keeps the draft for the next publish. + * Returns whether it deleted. + */ + remove(ref: HostedDraftRef, expectedEtag?: string | null): Promise; +} + +export function createDraftStore(deps: { + kv: KVStorage; + store: DeliveryStore; +}): DraftStore { + const { kv, store } = deps; + + const slugOf = async (ref: HostedDraftRef): Promise => { + const record = await kv.get(ref.organizationId, slugKey(ref)); + return typeof record?.slug === "string" ? record.slug : null; + }; + + const load = async (ref: HostedDraftRef): Promise => { + const slug = await slugOf(ref); + if (!slug) return null; + const object = await store.get(deliveryKeys.draft(ref.site, slug)); + if (!object) return { slug, body: emptyDraft(), etag: null }; + return { + slug, + body: parseDraftBody(JSON.parse(object.text)), + etag: object.etag, + }; + }; + + return { + load, + update: (ref, change) => + withDraftLock(`${ref.organizationId}\0${slugKey(ref)}`, async () => { + const current = await load(ref); + const slug = current?.slug ?? newDraftSlug(); + if (!current) { + await kv.set(ref.organizationId, slugKey(ref), { slug }); + } + const body = change(current?.body ?? emptyDraft()); + const { etag } = await store.putJson( + deliveryKeys.draft(ref.site, slug), + body, + CACHE_DRAFT, + ); + return { slug, body, etag }; + }), + remove: (ref, expectedEtag) => + withDraftLock(`${ref.organizationId}\0${slugKey(ref)}`, async () => { + const current = await load(ref); + if (!current) return false; + if (expectedEtag !== undefined && current.etag !== expectedEtag) { + return false; + } + await store.delete(deliveryKeys.draft(ref.site, current.slug)); + await kv.delete(ref.organizationId, slugKey(ref)); + return true; + }), + }; +}